@revideo/2d 0.4.7-two.1024 → 0.4.8-alpha.1032

package/src/lib/utils/video/parser/parser.ts

@@ -0,0 +1,256 @@
+import {createFile} from 'mp4box';
+import {FrameSampler} from './sampler';
+import {Segment} from './segment';
+import {MP4FileSink} from './sink';
+import {Edit, description, mp4BoxEditToEdit} from './utils';
+
+/**
+ * Loads the file at the given URI until it finds the moov box.
+ * Once found, it calls `setConfig` with the video configuration.
+ * @param uri - The URI of the video file.
+ * @param setConfig - Callback to set the video configuration.
+ * @returns
+ */
+async function getFileInfo(uri: string) {
+  return new Promise<{
+    file: any;
+    edits: Edit[];
+    config: VideoDecoderConfig;
+  }>((res, rej) => {
+    const file = createFile();
+    let found = false;
+    const controller = new AbortController();
+
+    file.onReady = (info: any) => {
+      found = true;
+      controller.abort();
+
+      const track = info.videoTracks[0];
+
+      const config = {
+        // Browser doesn't support parsing full vp8 codec (eg: `vp08.00.41.08`),
+        // they only support `vp8`.
+        codec: track.codec.startsWith('vp08') ? 'vp8' : track.codec,
+        codedHeight: track.video.height,
+        codedWidth: track.video.width,
+        description: description(file, track),
+      };
+
+      const defaultEdit: Omit<Edit, 'fps'> = {
+        mediaTime: 0,
+        segmentDuration: track.duration,
+        mediaRateInteger: 1,
+        mediaRateFraction: 0,
+      };
+
+      const editsFromVideo: Omit<Edit, 'fps'>[] = track.edits?.map(
+        (edit: any) => mp4BoxEditToEdit(edit),
+      );
+
+      // If there are no entries, use the default edit
+      const editsWithoutFps = editsFromVideo?.length
+        ? editsFromVideo
+        : [defaultEdit];
+
+      // Calculate FPS for each segment
+      const edits = editsWithoutFps.map(edit => {
+        const trackDurationInSec = track.duration / track.timescale;
+        const segmentDurationInSec =
+          edit.segmentDuration / track.movie_timescale;
+        const segmentFrames =
+          track.nb_samples * (segmentDurationInSec / trackDurationInSec);
+        const mediaRate =
+          edit.mediaRateInteger + edit.mediaRateFraction / 0xffff;
+        const fps = (segmentFrames / segmentDurationInSec) * mediaRate;
+        return {...edit, fps};
+      });
+
+      file.setExtractionOptions(track.id);
+      file.start();
+
+      res({file, edits, config});
+    };
+
+    return fetch(uri, {signal: controller.signal}).then(async response => {
+      if (!response.body) {
+        throw new Error('Response body is null');
+      }
+
+      const reader = response.body.getReader();
+      const sink = new MP4FileSink(file, () => {});
+
+      while (!found) {
+        await reader.read().then(({done, value}) => {
+          if (done) {
+            file.flush();
+            controller.abort();
+            rej('Could not find moov');
+            return;
+          }
+
+          sink.write(value);
+        });
+      }
+    });
+  });
+}
+
+export class Mp4Parser {
+  private uri: string;
+  private file: any;
+  private edits: Edit[] = [];
+  private decoderConfig?: VideoDecoderConfig;
+
+  private startTime: number;
+  private targetFps: number;
+
+  private nextSegment = 0;
+  private sampler?: FrameSampler;
+
+  public constructor(uri: string, targetFps: number, startTime: number) {
+    this.uri = uri;
+    this.targetFps = targetFps;
+    this.startTime = startTime;
+  }
+
+  public async start() {
+    const {file, edits, config} = await getFileInfo(this.uri);
+    this.file = file;
+    this.edits = edits;
+    this.decoderConfig = config;
+  }
+
+  public async getNextFrame() {
+    // Start the first segment
+    if (!this.sampler) {
+      // Skip segments until the start time
+      let startTimeWithinSegment = this.startTime;
+      while (this.nextSegment < this.edits.length) {
+        const segmentDurationInSeconds = this.getSecondDurationOfSegment(
+          this.edits[this.nextSegment],
+        );
+        if (startTimeWithinSegment < segmentDurationInSeconds) {
+          break;
+        }
+
+        startTimeWithinSegment -= segmentDurationInSeconds;
+        this.nextSegment++;
+      }
+
+      // The timestamp is outside of the video
+      if (this.nextSegment >= this.edits.length) {
+        throw new Error(
+          `Timestamp ${this.startTime} is outside of the video, max timestamp is ${this.getDuration()}`,
+        );
+      }
+
+      const segment = new Segment(
+        this.uri,
+        this.file,
+        this.edits,
+        this.nextSegment,
+        startTimeWithinSegment,
+      );
+      await segment.start(this.decoderConfig!);
+
+      this.sampler = new FrameSampler(
+        segment,
+        this.edits[this.nextSegment].fps,
+        this.targetFps,
+        0,
+      );
+      this.nextSegment++;
+    }
+
+    // Try to get the next frame
+    let frame = await this.sampler.getNextFrame();
+
+    // If there are no more frames in the current segment, start the next segment
+    while (!frame && this.nextSegment < this.edits.length) {
+      this.sampler.getSegment().close();
+      this.sampler.getLastFrame()?.close();
+      const segment = new Segment(
+        this.uri,
+        this.file,
+        this.edits,
+        this.nextSegment,
+        0,
+      );
+      await segment.start(this.decoderConfig!);
+
+      this.sampler = new FrameSampler(
+        segment,
+        this.edits[this.nextSegment].fps,
+        this.targetFps,
+        this.sampler.getSum(), // Carry over the sum from the previous segment
+      );
+      this.nextSegment++;
+
+      frame = await this.sampler.getNextFrame();
+    }
+
+    // If there are no more frames, return the last frame
+    if (!frame) {
+      frame = this.sampler.getLastFrame();
+
+      // We can close the segment
+      await this.sampler.getSegment().close();
+
+      // If we still don't have a frame, throw an error
+      if (!frame) {
+        throw new Error('There are no frames in the video.');
+      }
+    }
+
+    return frame;
+  }
+
+  private getSecondDurationOfSegment(edit: Edit) {
+    const mediaRate = edit.mediaRateInteger + edit.mediaRateFraction / 0xffff;
+    const duration =
+      edit.segmentDuration / this.file.getInfo().videoTracks[0].movie_timescale;
+    return duration / mediaRate;
+  }
+
+  public getDuration() {
+    return this.edits.reduce(
+      (acc, edit) => acc + this.getSecondDurationOfSegment(edit),
+      0,
+    );
+  }
+
+  private getTimeSubtractingFrames(frames: number) {
+    const durationOfPastSegmentsInSeconds = this.edits
+      .slice(0, this.nextSegment - 1)
+      .reduce((acc, edit) => acc + this.getSecondDurationOfSegment(edit), 0);
+
+    if (!this.sampler) {
+      throw new Error('No current segment');
+    }
+
+    const currentSegmentStartTime = this.sampler.getSegment().getStartTime();
+    const samplerTime = this.sampler.getTime(frames);
+
+    return (
+      durationOfPastSegmentsInSeconds + currentSegmentStartTime + samplerTime
+    );
+  }
+
+  public getTime() {
+    return this.getTimeSubtractingFrames(0);
+  }
+
+  public getLastTime() {
+    return this.getTimeSubtractingFrames(1);
+  }
+
+  public getLastFrame() {
+    return this.sampler?.getLastFrame();
+  }
+
+  public close() {
+    this.sampler?.getSegment().close();
+    this.sampler?.getLastFrame()?.close();
+    this.file.flush();
+  }
+}

package/src/lib/utils/video/parser/sampler.ts

@@ -0,0 +1,72 @@
+import {Segment} from './segment';
+
+export class FrameSampler {
+  private framesRequested = 0;
+  private lastFrame?: VideoFrame;
+
+  public constructor(
+    private segment: Segment,
+    private sourceFps: number,
+    private targetFps: number,
+    private sum: number = 0,
+  ) {}
+
+  public async getNextFrame() {
+    const samplingRate = this.sourceFps / this.targetFps;
+    this.sum += samplingRate;
+    this.framesRequested += 1;
+
+    // Return last frame if we haven't progressed enough to get a new frame.
+    if (this.sum < 1 && this.lastFrame) {
+      return this.lastFrame;
+    }
+
+    // Burn frames if we have too many.
+    while (this.sum >= 2) {
+      // Burn frame
+      const frame = await this.segment.getNextFrame();
+
+      // If there are no more frames, return
+      if (!frame) {
+        return;
+      }
+
+      frame.close();
+      this.sum -= 1;
+    }
+
+    // Get a new frame.
+    if (this.sum >= 1 || !this.lastFrame) {
+      this.sum -= 1;
+      const frame = await this.segment.getNextFrame();
+
+      // If there are no more frames, return
+      if (!frame) {
+        return;
+      }
+
+      this.lastFrame?.close();
+      this.lastFrame = frame;
+      return frame;
+    }
+
+    // One of the three if statements above is always true.
+    throw new Error('Unreachable code');
+  }
+
+  public getLastFrame() {
+    return this.lastFrame;
+  }
+
+  public getSum() {
+    return this.sum;
+  }
+
+  public getSegment() {
+    return this.segment;
+  }
+
+  public getTime(framesToSubtract: number) {
+    return (this.framesRequested - framesToSubtract) / this.targetFps;
+  }
+}

package/src/lib/utils/video/parser/segment.ts

@@ -0,0 +1,252 @@
+import {MP4FileSink} from './sink';
+import {Edit} from './utils';
+
+export class Segment {
+  private done: boolean = false;
+  private abortController = new AbortController();
+  private uri: string;
+
+  private file: any;
+  private edit: Edit;
+
+  private responseFinished: boolean = false;
+  private decoder: VideoDecoder;
+
+  private framesProcessed = 0;
+  private startTime: number;
+  private framesDue = 0;
+  private frameBuffer: VideoFrame[] = [];
+
+  private readMore: () => Promise<void> = async () => {};
+
+  public constructor(
+    uri: string,
+    file: any,
+    edits: Edit[],
+    currentSegment: number,
+    startTime: number = 0,
+  ) {
+    this.uri = uri;
+
+    this.file = file;
+    this.file.onSamples = this.onSamples.bind(this);
+    this.edit = edits[currentSegment];
+
+    this.startTime = startTime;
+
+    // Check how many frames we skipped because of the startTime and add them to the frames processed.
+    const framesSkipped = Math.floor(this.startTime * this.edit.fps);
+    this.framesProcessed += framesSkipped;
+
+    this.decoder = new VideoDecoder({
+      output: this.onFrame.bind(this),
+      error(e) {
+        console.error(e);
+      },
+    });
+  }
+
+  public async start(decoderConfig: VideoDecoderConfig) {
+    // If this is an empty edit, we just need to fill the buffer with empty frames.
+    if (this.edit.mediaTime === -1) {
+      this.done = true;
+      this.responseFinished = true;
+      this.decoder.close();
+
+      const segmentDurationInSeconds =
+        this.edit.segmentDuration /
+        this.file.getInfo().videoTracks[0].movie_timescale;
+      const framesToFill = segmentDurationInSeconds * this.edit.fps;
+
+      const height = this.file.getInfo().videoTracks[0].track_height;
+      const width = this.file.getInfo().videoTracks[0].track_width;
+
+      const bufferSize = height * width * 4;
+      const buffer = new ArrayBuffer(bufferSize);
+      const uint8Array = new Uint8Array(buffer);
+
+      // Make the frame black
+      uint8Array.fill(0);
+
+      this.frameBuffer = Array.from({length: framesToFill}, () => {
+        return new VideoFrame(uint8Array, {
+          timestamp: 0,
+          duration: 1 / this.edit.fps,
+          codedHeight: height,
+          codedWidth: width,
+          format: 'BGRA',
+        });
+      });
+
+      return;
+    }
+
+    this.decoder.configure(decoderConfig);
+    const videoTrack = this.file.getInfo().videoTracks[0];
+    const trak = this.file.getTrackById(videoTrack.id);
+
+    const mediaTimeInTimescale = this.edit.mediaTime;
+    const startTimeInTimescale = this.startTime * videoTrack.timescale;
+    const seekTimeInTimescale = mediaTimeInTimescale + startTimeInTimescale;
+
+    const seekTimeInSec = seekTimeInTimescale / videoTrack.timescale;
+    const seekInfo = this.file.seekTrack(seekTimeInSec, true, trak);
+    this.readMore = await this.startStreamingAtOffset(
+      this.file,
+      this.uri,
+      seekInfo.offset,
+    );
+  }
+
+  /**
+   * Starts streaming the video at the given URI from the given offset.
+   * @param file - MP4Box file. Needs to be created and configured before calling this function.
+   * @param uri - URI of the video file.
+   * @param offset - Offset to start streaming from.
+   * @returns - A function to read more data from the response.
+   */
+  private async startStreamingAtOffset(file: any, uri: string, offset: number) {
+    return fetch(uri, {
+      headers: {
+        /* eslint-disable-next-line @typescript-eslint/naming-convention */
+        Range: `bytes=${offset}-`,
+      },
+      signal: this.abortController.signal,
+    }).then(async response => {
+      if (!response.body) {
+        throw new Error('Response body is null');
+      }
+
+      const reader = response.body.getReader();
+      const sink = new MP4FileSink(file, () => {}, offset);
+
+      return async () => {
+        return reader.read().then(({done, value}) => {
+          // Request is done.
+          if (done) {
+            this.responseFinished = true;
+            this.abortController.abort();
+            this.decoder.flush();
+            sink.close();
+            return;
+          }
+
+          sink.write(value);
+        });
+      };
+    });
+  }
+
+  /**
+   * Called when samples are available on the MP4 file.
+   * Sends chunks to the decoder.
+   */
+  private onSamples(_unused1: any, _unused2: any, samples: any) {
+    for (const sample of samples) {
+      const chunk = new EncodedVideoChunk({
+        type: sample.is_sync ? 'key' : 'delta',
+        timestamp: (1e6 * sample.cts) / sample.timescale,
+        duration: (1e6 * sample.duration) / sample.timescale,
+        data: sample.data,
+      });
+      this.framesDue++;
+      this.decoder.decode(chunk);
+
+      const videoTrack = this.file.getInfo().videoTracks[0];
+      const trak = this.file.getTrackById(videoTrack.id);
+      this.file.releaseSample(trak, sample.number);
+    }
+  }
+
+  /**
+   * Called when the decoder has a frame ready.
+   * Pushes the frame to the buffer so it can be consumed.
+   */
+  private onFrame(frame: VideoFrame) {
+    this.framesDue--;
+
+    // If the frame comes before the seek time, close it.
+    const mediaTimeInSec =
+      this.edit.mediaTime / this.file.getInfo().videoTracks[0].timescale;
+    const seekTimeInSec = this.startTime + mediaTimeInSec;
+    const frameTimeInSec = frame.timestamp / 1e6;
+    if (frameTimeInSec < seekTimeInSec) {
+      frame.close();
+      return;
+    }
+
+    // Check if we are past the segment duration.
+    const segmentDurationInSec =
+      this.edit.segmentDuration /
+      this.file.getInfo().videoTracks[0].movie_timescale;
+    const segmentEndTime = mediaTimeInSec + segmentDurationInSec;
+    if (frameTimeInSec > segmentEndTime) {
+      frame.close();
+      this.done = true;
+      this.decoder.flush();
+      return;
+    }
+
+    this.frameBuffer.push(frame);
+  }
+
+  private async populateBuffer() {
+    // Fetch more frames if we don't have any.
+    while (this.frameBuffer.length === 0 && !this.responseFinished) {
+      await this.readMore();
+      await new Promise(res => setTimeout(res, 0));
+    }
+
+    // Wait for decoder if there are frames due.
+    if (this.frameBuffer.length === 0 && this.framesDue > 0) {
+      let maxIterations = 1000;
+      while (this.frameBuffer.length === 0) {
+        await new Promise(res => setTimeout(res, 10));
+        maxIterations--;
+
+        if (this.done) {
+          return;
+        }
+
+        if (maxIterations === 0) {
+          throw new Error(
+            `Timed out while waiting for VideoDecoder to produce a frame. Frames due: ${this.framesDue}`,
+          );
+        }
+      }
+    }
+  }
+
+  public async getNextFrame() {
+    await this.populateBuffer();
+    const frame = this.frameBuffer.shift();
+    if (frame) {
+      this.framesProcessed++;
+    }
+    return frame;
+  }
+
+  /**
+   * Called when we are done with the extractor.
+   */
+  public async close() {
+    this.abortController.abort();
+    this.frameBuffer.forEach(frame => frame.close());
+    try {
+      if (this.decoder.state === 'configured') {
+        await this.decoder.flush();
+        this.decoder.close();
+      }
+    } catch (e) {
+      // Ignore
+    }
+  }
+
+  public getFramesProcessed() {
+    return this.framesProcessed;
+  }
+
+  public getStartTime() {
+    return this.startTime;
+  }
+}

package/src/lib/utils/video/parser/sink.ts

@@ -0,0 +1,29 @@
+// Wraps an MP4Box File as a WritableStream underlying sink.
+export class MP4FileSink {
+  private setStatus;
+  private file;
+  private offset;
+
+  public constructor(file: any, setStatus: any, offset: number = 0) {
+    this.file = file;
+    this.setStatus = setStatus;
+    this.offset = offset;
+  }
+
+  public write(chunk: any) {
+    const buffer = new ArrayBuffer(chunk.byteLength);
+    new Uint8Array(buffer).set(chunk);
+
+    // Inform MP4Box where in the file this chunk is from.
+    (buffer as any).fileStart = this.offset;
+    this.offset += buffer.byteLength;
+
+    this.setStatus('fetch', (this.offset / 1024 ** 2).toFixed(1) + ' MiB');
+    this.file.appendBuffer(buffer);
+  }
+
+  public close() {
+    this.setStatus('fetch', 'Done');
+    this.file.flush();
+  }
+}

package/src/lib/utils/video/parser/utils.ts

@@ -0,0 +1,31 @@
+import {DataStream} from 'mp4box';
+
+export function description(file: any, track: any) {
+  const trak = file.getTrackById(track.id);
+  for (const entry of trak.mdia.minf.stbl.stsd.entries) {
+    const box = entry.avcC || entry.hvcC || entry.vpcC || entry.av1C;
+    if (box) {
+      const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
+      box.write(stream);
+      return new Uint8Array(stream.buffer, 8); // Remove the box header.
+    }
+  }
+  throw new Error('avcC, hvcC, vpcC, or av1C box not found');
+}
+
+export interface Edit {
+  mediaTime: number;
+  segmentDuration: number;
+  mediaRateInteger: number;
+  mediaRateFraction: number;
+  fps: number;
+}
+
+export function mp4BoxEditToEdit(edit: any): Omit<Edit, 'fps'> {
+  return {
+    mediaTime: edit.media_time,
+    segmentDuration: edit.segment_duration,
+    mediaRateInteger: edit.media_rate_integer,
+    mediaRateFraction: edit.media_rate_fraction,
+  };
+}

package/lib/components/HlsVideo.d.ts

@@ -1,62 +0,0 @@
-import {SerializedVector2, SignalValue, SimpleSignal} from '@revideo/core';
-import {DesiredLength} from '../partials';
-import {Media, MediaProps} from './Media';
-export interface HlsVideoProps extends MediaProps {
-  /**
-   * {@inheritDoc Video.alpha}
-   */
-  alpha?: SignalValue<number>;
-  /**
-   * {@inheritDoc Video.smoothing}
-   */
-  smoothing?: SignalValue<boolean>;
-  /**
-   * {@inheritDoc Video.png}
-   */
-  png?: SignalValue<boolean>;
-}
-export declare class HlsVideo extends Media {
-  /**
-   * The alpha value of this video.
-   *
-   * @remarks
-   * Unlike opacity, the alpha value affects only the video itself, leaving the
-   * fill, stroke, and children intact.
-   */
-  readonly alpha: SimpleSignal<number, this>;
-  /**
-   * Whether the video should be smoothed.
-   *
-   * @remarks
-   * When disabled, the video will be scaled using the nearest neighbor
-   * interpolation with no smoothing. The resulting video will appear pixelated.
-   *
-   * @defaultValue true
-   */
-  readonly smoothing: SimpleSignal<boolean, this>;
-  /**
-   * Whether the video frames should be extracted as PNGs. Uses JPEGs when
-   * set to false.
-   *
-   * @remarks
-   * PNGs have better image quality and support transparency, but they make
-   * rendering slower.
-   *
-   * @defaultValue false
-   */
-  readonly png: SimpleSignal<boolean, this>;
-  private static readonly pool;
-  constructor(props: HlsVideoProps);
-  protected desiredSize(): SerializedVector2<DesiredLength>;
-  protected mediaElement(): HTMLVideoElement;
-  protected seekedMedia(): HTMLVideoElement;
-  protected fastSeekedMedia(): HTMLVideoElement;
-  private video;
-  protected seekedVideo(): HTMLVideoElement;
-  protected fastSeekedVideo(): HTMLVideoElement;
-  protected lastFrame: HTMLImageElement | null;
-  protected seekFunction(): Promise<HTMLVideoElement>;
-  protected draw(context: CanvasRenderingContext2D): Promise<void>;
-  protected applyFlex(): void;
-}
-//# sourceMappingURL=HlsVideo.d.ts.map

package/lib/components/HlsVideo.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"HlsVideo.d.ts","sourceRoot":"","sources":["../../src/lib/components/HlsVideo.ts"],"names":[],"mappings":"AAAA,OAAO,EAIH,iBAAiB,EACjB,WAAW,EACX,YAAY,EAEb,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAC,aAAa,EAAC,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAC,KAAK,EAAE,UAAU,EAAC,MAAM,SAAS,CAAC;AAG1C,MAAM,WAAW,aAAc,SAAQ,UAAU;IAC/C;;OAEG;IACH,KAAK,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC5B;;OAEG;IACH,SAAS,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IACjC;;OAEG;IACH,GAAG,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;CAC5B;AAED,qBACa,QAAS,SAAQ,KAAK;IACjC;;;;;;OAMG;IACH,SAEwB,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAE1D;;;;;;;;OAQG;IACH,SAEwB,SAAS,EAAE,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAE/D;;;;;;;;;OASG;IACH,SAEwB,GAAG,EAAE,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAEzD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAwC;gBAEjD,KAAK,EAAE,aAAa;cAIpB,WAAW,IAAI,iBAAiB,CAAC,aAAa,CAAC;IAalE,SAAS,CAAC,YAAY,IAAI,gBAAgB;IAI1C,SAAS,CAAC,WAAW,IAAI,gBAAgB;IAIzC,SAAS,CAAC,eAAe,IAAI,gBAAgB;IAK7C,OAAO,CAAC,KAAK;IAsCb,SAAS,CAAC,WAAW,IAAI,gBAAgB;IAoBzC,SAAS,CAAC,eAAe,IAAI,gBAAgB;IA+B7C,SAAS,CAAC,SAAS,EAAE,gBAAgB,GAAG,IAAI,CAAQ;cAGpC,YAAY;cAgBH,IAAI,CAAC,OAAO,EAAE,wBAAwB;cAwB5C,SAAS;CAO7B"}