@twick/2d 0.14.21 → 0.15.0

package/src/lib/utils/video/mp4-parser-manager.ts

@@ -1,72 +0,0 @@
-import {Mp4Parser} from './parser';
-
-// List of VideoFrameExtractors
-const videoFrameExtractors = new Map<string, Mp4Parser>();
-
-export async function dropExtractor(id: string, filePath: string) {
-  const extractorId = filePath + '-' + id;
-  const extractor = videoFrameExtractors.get(extractorId);
-  if (extractor) {
-    extractor.close();
-    videoFrameExtractors.delete(extractorId);
-  }
-}
-
-export async function getFrame(
-  id: string,
-  filePath: string,
-  time: number,
-  fps: number,
-) {
-  // Check if we already have a VideoFrameExtractor for this video
-  const extractorId = filePath + '-' + id;
-  let extractor = videoFrameExtractors.get(extractorId);
-
-  const frameDuration = 1 / fps;
-
-  /**
-   * Sometimes, HTMLVideoElement.duration is not accurate, which can lead to the
-   * requested time being greater than the duration.
-   * To prevent this, we clamp the time to the duration reported by the extractor.
-   */
-  const duration = extractor?.getDuration();
-  if (duration && time > duration) {
-    time = duration;
-  }
-
-  const isOldFrame =
-    extractor && Math.abs(time - extractor.getLastTime()) < frameDuration / 2;
-
-  // If time has not changed, return the last frame
-  if (extractor && isOldFrame) {
-    const lastFrame = extractor.getLastFrame();
-    if (!lastFrame) {
-      throw new Error('No last frame');
-    }
-
-    return lastFrame;
-  }
-
-  // If the video has skipped back we need to create a new extractor
-  if (extractor && time + frameDuration < extractor.getTime()) {
-    extractor.close();
-    videoFrameExtractors.delete(extractorId);
-    extractor = undefined;
-  }
-
-  // If the video has skipped forward we need to create a new extractor
-  if (extractor && time > extractor.getTime() + frameDuration * 1.5) {
-    extractor.close();
-    videoFrameExtractors.delete(extractorId);
-    extractor = undefined;
-  }
-
-  if (!extractor) {
-    extractor = new Mp4Parser(filePath, fps, time);
-    await extractor.start();
-    videoFrameExtractors.set(extractorId, extractor);
-  }
-
-  // Go to the frame that is closest to the requested time
-  return extractor.getNextFrame();
-}

package/src/lib/utils/video/parser/index.ts

@@ -1 +0,0 @@
-export * from './parser';

package/src/lib/utils/video/parser/parser.ts

@@ -1,257 +0,0 @@
-import {createFile} from 'mp4box';
-import {FrameSampler} from './sampler';
-import {Segment} from './segment';
-import {MP4FileSink} from './sink';
-import type {Edit} from './utils';
-import {description, mp4BoxEditToEdit} from './utils';
-
-/**
- * Loads the file at the given URI until it finds the moov box.
- * Once found, it calls `setConfig` with the video configuration.
- * @param uri - The URI of the video file.
- * @param setConfig - Callback to set the video configuration.
- * @returns
- */
-async function getFileInfo(uri: string) {
-  return new Promise<{
-    file: any;
-    edits: Edit[];
-    config: VideoDecoderConfig;
-  }>((res, rej) => {
-    const file = createFile();
-    let found = false;
-    const controller = new AbortController();
-
-    file.onReady = (info: any) => {
-      found = true;
-      controller.abort();
-
-      const track = info.videoTracks[0];
-
-      const config = {
-        // Browser doesn't support parsing full vp8 codec (eg: `vp08.00.41.08`),
-        // they only support `vp8`.
-        codec: track.codec.startsWith('vp08') ? 'vp8' : track.codec,
-        codedHeight: track.video.height,
-        codedWidth: track.video.width,
-        description: description(file, track),
-      };
-
-      const defaultEdit: Omit<Edit, 'fps'> = {
-        mediaTime: 0,
-        segmentDuration: track.duration,
-        mediaRateInteger: 1,
-        mediaRateFraction: 0,
-      };
-
-      const editsFromVideo: Omit<Edit, 'fps'>[] = track.edits?.map(
-        (edit: any) => mp4BoxEditToEdit(edit),
-      );
-
-      // If there are no entries, use the default edit
-      const editsWithoutFps = editsFromVideo?.length
-        ? editsFromVideo
-        : [defaultEdit];
-
-      // Calculate FPS for each segment
-      const edits = editsWithoutFps.map(edit => {
-        const trackDurationInSec = track.duration / track.timescale;
-        const segmentDurationInSec =
-          edit.segmentDuration / track.movie_timescale;
-        const segmentFrames =
-          track.nb_samples * (segmentDurationInSec / trackDurationInSec);
-        const mediaRate =
-          edit.mediaRateInteger + edit.mediaRateFraction / 0xffff;
-        const fps = (segmentFrames / segmentDurationInSec) * mediaRate;
-        return {...edit, fps};
-      });
-
-      file.setExtractionOptions(track.id);
-      file.start();
-
-      res({file, edits, config});
-    };
-
-    return fetch(uri, {signal: controller.signal}).then(async response => {
-      if (!response.body) {
-        throw new Error('Response body is null');
-      }
-
-      const reader = response.body.getReader();
-      const sink = new MP4FileSink(file, () => {});
-
-      while (!found) {
-        await reader.read().then(({done, value}) => {
-          if (done) {
-            file.flush();
-            controller.abort();
-            rej('Could not find moov');
-            return;
-          }
-
-          sink.write(value);
-        });
-      }
-    });
-  });
-}
-
-export class Mp4Parser {
-  private uri: string;
-  private file: any;
-  private edits: Edit[] = [];
-  private decoderConfig?: VideoDecoderConfig;
-
-  private startTime: number;
-  private targetFps: number;
-
-  private nextSegment = 0;
-  private sampler?: FrameSampler;
-
-  public constructor(uri: string, targetFps: number, startTime: number) {
-    this.uri = uri;
-    this.targetFps = targetFps;
-    this.startTime = startTime;
-  }
-
-  public async start() {
-    const {file, edits, config} = await getFileInfo(this.uri);
-    this.file = file;
-    this.edits = edits;
-    this.decoderConfig = config;
-  }
-
-  public async getNextFrame() {
-    // Start the first segment
-    if (!this.sampler) {
-      // Skip segments until the start time
-      let startTimeWithinSegment = this.startTime;
-      while (this.nextSegment < this.edits.length) {
-        const segmentDurationInSeconds = this.getSecondDurationOfSegment(
-          this.edits[this.nextSegment],
-        );
-        if (startTimeWithinSegment < segmentDurationInSeconds) {
-          break;
-        }
-
-        startTimeWithinSegment -= segmentDurationInSeconds;
-        this.nextSegment++;
-      }
-
-      // The timestamp is outside of the video
-      if (this.nextSegment >= this.edits.length) {
-        throw new Error(
-          `Timestamp ${this.startTime} is outside of the video, max timestamp is ${this.getDuration()}`,
-        );
-      }
-
-      const segment = new Segment(
-        this.uri,
-        this.file,
-        this.edits,
-        this.nextSegment,
-        startTimeWithinSegment,
-      );
-      await segment.start(this.decoderConfig!);
-
-      this.sampler = new FrameSampler(
-        segment,
-        this.edits[this.nextSegment].fps,
-        this.targetFps,
-        0,
-      );
-      this.nextSegment++;
-    }
-
-    // Try to get the next frame
-    let frame = await this.sampler.getNextFrame();
-
-    // If there are no more frames in the current segment, start the next segment
-    while (!frame && this.nextSegment < this.edits.length) {
-      this.sampler.getSegment().close();
-      this.sampler.getLastFrame()?.close();
-      const segment = new Segment(
-        this.uri,
-        this.file,
-        this.edits,
-        this.nextSegment,
-        0,
-      );
-      await segment.start(this.decoderConfig!);
-
-      this.sampler = new FrameSampler(
-        segment,
-        this.edits[this.nextSegment].fps,
-        this.targetFps,
-        this.sampler.getSum(), // Carry over the sum from the previous segment
-      );
-      this.nextSegment++;
-
-      frame = await this.sampler.getNextFrame();
-    }
-
-    // If there are no more frames, return the last frame
-    if (!frame) {
-      frame = this.sampler.getLastFrame();
-
-      // We can close the segment
-      await this.sampler.getSegment().close();
-
-      // If we still don't have a frame, throw an error
-      if (!frame) {
-        throw new Error('There are no frames in the video.');
-      }
-    }
-
-    return frame;
-  }
-
-  private getSecondDurationOfSegment(edit: Edit) {
-    const mediaRate = edit.mediaRateInteger + edit.mediaRateFraction / 0xffff;
-    const duration =
-      edit.segmentDuration / this.file.getInfo().videoTracks[0].movie_timescale;
-    return duration / mediaRate;
-  }
-
-  public getDuration() {
-    return this.edits.reduce(
-      (acc, edit) => acc + this.getSecondDurationOfSegment(edit),
-      0,
-    );
-  }
-
-  private getTimeSubtractingFrames(frames: number) {
-    const durationOfPastSegmentsInSeconds = this.edits
-      .slice(0, this.nextSegment - 1)
-      .reduce((acc, edit) => acc + this.getSecondDurationOfSegment(edit), 0);
-
-    if (!this.sampler) {
-      throw new Error('No current segment');
-    }
-
-    const currentSegmentStartTime = this.sampler.getSegment().getStartTime();
-    const samplerTime = this.sampler.getTime(frames);
-
-    return (
-      durationOfPastSegmentsInSeconds + currentSegmentStartTime + samplerTime
-    );
-  }
-
-  public getTime() {
-    return this.getTimeSubtractingFrames(0);
-  }
-
-  public getLastTime() {
-    return this.getTimeSubtractingFrames(1);
-  }
-
-  public getLastFrame() {
-    return this.sampler?.getLastFrame();
-  }
-
-  public close() {
-    this.sampler?.getSegment().close();
-    this.sampler?.getLastFrame()?.close();
-    this.file.flush();
-  }
-}

package/src/lib/utils/video/parser/sampler.ts

@@ -1,72 +0,0 @@
-import type {Segment} from './segment';
-
-export class FrameSampler {
-  private framesRequested = 0;
-  private lastFrame?: VideoFrame;
-
-  public constructor(
-    private segment: Segment,
-    private sourceFps: number,
-    private targetFps: number,
-    private sum: number = 0,
-  ) {}
-
-  public async getNextFrame() {
-    const samplingRate = this.sourceFps / this.targetFps;
-    this.sum += samplingRate;
-    this.framesRequested += 1;
-
-    // Return last frame if we haven't progressed enough to get a new frame.
-    if (this.sum < 1 && this.lastFrame) {
-      return this.lastFrame;
-    }
-
-    // Burn frames if we have too many.
-    while (this.sum >= 2) {
-      // Burn frame
-      const frame = await this.segment.getNextFrame();
-
-      // If there are no more frames, return
-      if (!frame) {
-        return;
-      }
-
-      frame.close();
-      this.sum -= 1;
-    }
-
-    // Get a new frame.
-    if (this.sum >= 1 || !this.lastFrame) {
-      this.sum -= 1;
-      const frame = await this.segment.getNextFrame();
-
-      // If there are no more frames, return
-      if (!frame) {
-        return;
-      }
-
-      this.lastFrame?.close();
-      this.lastFrame = frame;
-      return frame;
-    }
-
-    // One of the three if statements above is always true.
-    throw new Error('Unreachable code');
-  }
-
-  public getLastFrame() {
-    return this.lastFrame;
-  }
-
-  public getSum() {
-    return this.sum;
-  }
-
-  public getSegment() {
-    return this.segment;
-  }
-
-  public getTime(framesToSubtract: number) {
-    return (this.framesRequested - framesToSubtract) / this.targetFps;
-  }
-}

package/src/lib/utils/video/parser/segment.ts

@@ -1,302 +0,0 @@
-import {MP4FileSink} from './sink';
-import type {Edit} from './utils';
-
-const MAX_DECODE_QUEUE_SIZE = 30;
-
-export class Segment {
-  private done: boolean = false;
-  private currentFramePastSegmentEndTime: boolean = false;
-  private abortController = new AbortController();
-  private uri: string;
-
-  private file: any;
-  private edit: Edit;
-
-  private responseFinished: boolean = false;
-  private decoder: VideoDecoder;
-
-  private framesProcessed = 0;
-  private startTime: number;
-  private framesDue = 0;
-  private frameBuffer: VideoFrame[] = [];
-  private encodedChunkQueue: EncodedVideoChunk[] = [];
-
-  private readMore: () => Promise<void> = async () => {};
-
-  public constructor(
-    uri: string,
-    file: any,
-    edits: Edit[],
-    currentSegment: number,
-    startTime: number = 0,
-  ) {
-    this.uri = uri;
-
-    this.file = file;
-    this.file.onSamples = this.onSamples.bind(this);
-    this.edit = edits[currentSegment];
-
-    this.startTime = startTime;
-
-    // Check how many frames we skipped because of the startTime and add them to the frames processed.
-    const framesSkipped = Math.floor(this.startTime * this.edit.fps);
-    this.framesProcessed += framesSkipped;
-
-    this.decoder = new VideoDecoder({
-      output: this.onFrame.bind(this),
-      error(e) {
-        console.error(e);
-      },
-    });
-  }
-
-  public async start(decoderConfig: VideoDecoderConfig) {
-    // If this is an empty edit, we just need to fill the buffer with empty frames.
-    if (this.edit.mediaTime === -1) {
-      this.done = true;
-      this.responseFinished = true;
-      this.decoder.close();
-
-      const segmentDurationInSeconds =
-        this.edit.segmentDuration /
-        this.file.getInfo().videoTracks[0].movie_timescale;
-      const framesToFill = Math.ceil(segmentDurationInSeconds * this.edit.fps);
-
-      const height = this.file.getInfo().videoTracks[0].track_height;
-      const width = this.file.getInfo().videoTracks[0].track_width;
-
-      const bufferSize = height * width * 4;
-      const buffer = new ArrayBuffer(bufferSize);
-      const uint8Array = new Uint8Array(buffer);
-
-      // Make the frame black
-      uint8Array.fill(0);
-
-      this.frameBuffer = Array.from({length: framesToFill}, () => {
-        return new VideoFrame(uint8Array, {
-          timestamp: 0,
-          duration: 1 / this.edit.fps,
-          codedHeight: height,
-          codedWidth: width,
-          format: 'BGRA',
-        });
-      });
-
-      return;
-    }
-
-    this.decoder.configure(decoderConfig);
-    const videoTrack = this.file.getInfo().videoTracks[0];
-    const trak = this.file.getTrackById(videoTrack.id);
-
-    const mediaTimeInTimescale = this.edit.mediaTime;
-    const startTimeInTimescale = this.startTime * videoTrack.timescale;
-    const seekTimeInTimescale = mediaTimeInTimescale + startTimeInTimescale;
-
-    const seekTimeInSec = seekTimeInTimescale / videoTrack.timescale;
-    const seekInfo = this.file.seekTrack(seekTimeInSec, true, trak);
-    this.readMore = await this.startStreamingAtOffset(
-      this.file,
-      this.uri,
-      seekInfo.offset,
-    );
-  }
-
-  /**
-   * Starts streaming the video at the given URI from the given offset.
-   * @param file - MP4Box file. Needs to be created and configured before calling this function.
-   * @param uri - URI of the video file.
-   * @param offset - Offset to start streaming from.
-   * @returns - A function to read more data from the response.
-   */
-  private async startStreamingAtOffset(file: any, uri: string, offset: number) {
-    return fetch(uri, {
-      headers: {
-        /* eslint-disable-next-line @typescript-eslint/naming-convention */
-        Range: `bytes=${offset}-`,
-      },
-      signal: this.abortController.signal,
-    }).then(async response => {
-      if (!response.body) {
-        throw new Error('Response body is null');
-      }
-
-      const reader = response.body.getReader();
-      const sink = new MP4FileSink(file, () => {}, offset);
-
-      return async () => {
-        return reader.read().then(({done, value}) => {
-          // Request is done.
-          if (done) {
-            this.responseFinished = true;
-            this.abortController.abort();
-            sink.close();
-            return;
-          }
-
-          sink.write(value);
-        });
-      };
-    });
-  }
-
-  /**
-   * Called when samples are available on the MP4 file.
-   * Sends chunks to the decoder.
-   */
-  private onSamples(_unused1: any, _unused2: any, samples: any) {
-    for (const sample of samples) {
-      const chunk = new EncodedVideoChunk({
-        type: sample.is_sync ? 'key' : 'delta',
-        timestamp: (1e6 * sample.cts) / sample.timescale,
-        duration: (1e6 * sample.duration) / sample.timescale,
-        data: sample.data,
-      });
-      this.framesDue++;
-      this.encodedChunkQueue.push(chunk);
-
-      const videoTrack = this.file.getInfo().videoTracks[0];
-      const trak = this.file.getTrackById(videoTrack.id);
-      this.file.releaseSample(trak, sample.number);
-    }
-  }
-
-  private async decodeChunks() {
-    while (
-      this.encodedChunkQueue.length > 0 &&
-      this.decoder.decodeQueueSize < MAX_DECODE_QUEUE_SIZE
-    ) {
-      const chunk = this.encodedChunkQueue.shift();
-      if (chunk) {
-        this.decoder.decode(chunk);
-      }
-    }
-    // When edit is empty, we cannot call decoder.flush() because decoder was already closed
-    if (this.done) {
-      this.currentFramePastSegmentEndTime = true;
-      return;
-    }
-    if (this.responseFinished && this.encodedChunkQueue.length === 0) {
-      await this.flushDecoderWithRetry();
-      this.currentFramePastSegmentEndTime = true;
-      return;
-    }
-  }
-
-  private async flushDecoderWithRetry(maxRetries = 3, timeoutMs = 2000) {
-    for (let attempt = 0; attempt < maxRetries; attempt++) {
-      // Sometimes decoder.flush does not resolve even though the queue size is zero
-      if (this.decoder.decodeQueueSize === 0) {
-        return;
-      }
-      try {
-        await Promise.race([
-          this.decoder.flush(),
-          new Promise((_, reject) =>
-            setTimeout(() => reject(new Error('Flush timeout')), timeoutMs),
-          ),
-        ]);
-      } catch (error: any) {
-        if (attempt === maxRetries) {
-          throw error;
-        }
-      }
-    }
-  }
-
-  /**
-   * Called when the decoder has a frame ready.
-   * Pushes the frame to the buffer so it can be consumed.
-   */
-  private async onFrame(frame: VideoFrame) {
-    this.framesDue--;
-
-    // If the frame comes before the seek time, close it.
-    const mediaTimeInSec =
-      this.edit.mediaTime / this.file.getInfo().videoTracks[0].timescale;
-    const seekTimeInSec = this.startTime + mediaTimeInSec;
-    const frameTimeInSec = frame.timestamp / 1e6;
-    if (frameTimeInSec < seekTimeInSec) {
-      frame.close();
-      return;
-    }
-
-    // Check if we are past the segment duration.
-    const segmentDurationInSec =
-      this.edit.segmentDuration /
-      this.file.getInfo().videoTracks[0].movie_timescale;
-    const segmentEndTime = mediaTimeInSec + segmentDurationInSec;
-    if (frameTimeInSec > segmentEndTime) {
-      frame.close();
-      this.done = true;
-      await this.decoder.flush();
-      return;
-    }
-
-    this.frameBuffer.push(frame);
-  }
-
-  private async populateBuffer() {
-    // Fetch more frames if we don't have any.
-    while (
-      this.frameBuffer.length === 0 &&
-      !this.currentFramePastSegmentEndTime
-    ) {
-      if (!this.responseFinished) {
-        await this.readMore();
-      }
-      await this.decodeChunks();
-      await new Promise(res => setTimeout(res, 0));
-    }
-
-    // Wait for decoder if there are frames due.
-    if (this.frameBuffer.length === 0 && this.framesDue > 0) {
-      let maxIterations = 200;
-      while (this.frameBuffer.length === 0) {
-        await new Promise(res => setTimeout(res, 10));
-        maxIterations--;
-
-        if (this.done) {
-          return;
-        }
-
-        if (maxIterations === 0) {
-          return; // TODO: investigate further, shouldn't this be an error?
-        }
-      }
-    }
-  }
-
-  public async getNextFrame() {
-    await this.populateBuffer();
-    const frame = this.frameBuffer.shift();
-    if (frame) {
-      this.framesProcessed++;
-    }
-    return frame;
-  }
-
-  /**
-   * Called when we are done with the extractor.
-   */
-  public async close() {
-    this.abortController.abort();
-    this.frameBuffer.forEach(frame => frame.close());
-    try {
-      if (this.decoder.state === 'configured') {
-        await this.decoder.flush();
-        this.decoder.close();
-      }
-    } catch (e) {
-      // Ignore
-    }
-  }
-
-  public getFramesProcessed() {
-    return this.framesProcessed;
-  }
-
-  public getStartTime() {
-    return this.startTime;
-  }
-}