@revideo/2d 0.4.3 → 0.4.5-alpha.1013

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@revideo/2d",
-  "version": "0.4.3",
+  "version": "0.4.5-alpha.1013+946fa9ea",
   "description": "A 2D renderer for revideo",
   "author": "revideo",
   "homepage": "https://re.video/",
@@ -29,8 +29,8 @@
   ],
   "devDependencies": {
     "@preact/signals": "^1.2.1",
-    "@revideo/internal": "0.0.0",
-    "@revideo/ui": "0.4.3",
+    "@revideo/internal": "^0.0.0",
+    "@revideo/ui": "^0.4.5-alpha.1013+946fa9ea",
     "clsx": "^2.0.0",
     "jsdom": "^22.1.0",
     "preact": "^10.19.2",
@@ -40,12 +40,12 @@
     "@codemirror/language": "^6.10.1",
     "@lezer/common": "^1.2.1",
     "@lezer/highlight": "^1.2.0",
-    "@revideo/core": "0.4.3",
+    "@revideo/core": "^0.4.5-alpha.1013+946fa9ea",
     "code-fns": "^0.8.2",
     "hls.js": "^1.5.11",
     "mathjax-full": "^3.2.2",
     "mp4box": "^0.5.2",
     "parse-svg-path": "^0.1.2"
   },
-  "gitHead": "31ac96141a6a1bf1f1de519dc0475a0d8a8d0347"
+  "gitHead": "946fa9ead6ec3af889f9c013f72520a208cf3da6"
 }

package/src/lib/components/Video.ts

@@ -81,7 +81,9 @@ export class Video extends Media {
     super(props);
 
     // If the file is not mp4, warn.
-    if (!this.fullSource().endsWith('.mp4')) {
+    // strip the query string from the source
+    const src = this.fullSource().split('?')[0];
+    if (!src.endsWith('.mp4')) {
       console.warn(
         `WARNING: Video source is not an MP4 file. This may significantly slow down rendering: ${this.fullSource()}`,
       );

package/src/lib/utils/video/ffmpeg-client.ts

@@ -5,7 +5,7 @@ export class ImageCommunication {
     }
 
     import.meta.hot.on(
-      'revideo:ffmpeg-video-frame-res',
+      'revideo:ffmpeg-decoder:video-frame-res',
       this.handler.bind(this),
     );
   }
@@ -50,14 +50,13 @@ export class ImageCommunication {
 
       this.nextFrameHandlers.push(handler);
 
-      import.meta.hot.send('revideo:ffmpeg-video-frame', {
+      import.meta.hot.send('revideo:ffmpeg-decoder:video-frame', {
         data: {
           id: id,
           filePath: src,
           startTime: time,
           duration,
           fps,
-          png: true,
         },
       });
     });

package/src/lib/utils/video/mp4-parser-manager.ts

@@ -1,7 +1,7 @@
-import {FrameExtractor} from './mp4-parser';
+import {Mp4Parser} from './parser';
 
 // List of VideoFrameExtractors
-const videoFrameExtractors = new Map<string, FrameExtractor>();
+const videoFrameExtractors = new Map<string, Mp4Parser>();
 
 export async function getFrame(
   id: string,
@@ -15,12 +15,22 @@ export async function getFrame(
 
   const frameDuration = 1 / fps;
 
+  /**
+   * Sometimes, HTMLVideoElement.duration is not accurate, which can lead to the
+   * requested time being greater than the duration.
+   * To prevent this, we clamp the time to the duration reported by the extractor.
+   */
+  const duration = extractor?.getDuration();
+  if (duration && time > duration) {
+    time = duration;
+  }
+
   const isOldFrame =
     extractor && Math.abs(time - extractor.getLastTime()) < frameDuration / 2;
 
   // If time has not changed, return the last frame
-  if (isOldFrame) {
-    const lastFrame = extractor!.getLastFrame();
+  if (extractor && isOldFrame) {
+    const lastFrame = extractor.getLastFrame();
     if (!lastFrame) {
       throw new Error('No last frame');
     }
@@ -43,7 +53,7 @@ export async function getFrame(
   }
 
   if (!extractor) {
-    extractor = new FrameExtractor(filePath, fps, time);
+    extractor = new Mp4Parser(filePath, fps, time);
     await extractor.start();
     videoFrameExtractors.set(extractorId, extractor);
   }

package/src/lib/utils/video/parser/index.ts

@@ -0,0 +1 @@
+export * from './parser';

package/src/lib/utils/video/parser/parser.ts

@@ -0,0 +1,247 @@
+import {createFile} from 'mp4box';
+import {FrameSampler} from './sampler';
+import {Segment} from './segment';
+import {MP4FileSink} from './sink';
+import {Edit, description, mp4BoxEditToEdit} from './utils';
+
+/**
+ * Loads the file at the given URI until it finds the moov box.
+ * Once found, it calls `setConfig` with the video configuration.
+ * @param uri - The URI of the video file.
+ * @param setConfig - Callback to set the video configuration.
+ * @returns
+ */
+async function getFileInfo(uri: string) {
+  return new Promise<{
+    file: any;
+    edits: Edit[];
+    config: VideoDecoderConfig;
+  }>((res, rej) => {
+    const file = createFile();
+    let found = false;
+
+    file.onReady = (info: any) => {
+      found = true;
+
+      const track = info.videoTracks[0];
+
+      const config = {
+        // Browser doesn't support parsing full vp8 codec (eg: `vp08.00.41.08`),
+        // they only support `vp8`.
+        codec: track.codec.startsWith('vp08') ? 'vp8' : track.codec,
+        codedHeight: track.video.height,
+        codedWidth: track.video.width,
+        description: description(file, track),
+      };
+
+      const defaultEdit: Omit<Edit, 'fps'> = {
+        mediaTime: 0,
+        segmentDuration: track.duration,
+        mediaRateInteger: 1,
+        mediaRateFraction: 0,
+      };
+
+      const editsFromVideo: Omit<Edit, 'fps'>[] = track.edits?.map(
+        (edit: any) => mp4BoxEditToEdit(edit),
+      );
+
+      // If there are no entries, use the default edit
+      const editsWithoutFps = editsFromVideo.length
+        ? editsFromVideo
+        : [defaultEdit];
+
+      // Calculate FPS for each segment
+      const edits = editsWithoutFps.map(edit => {
+        const trackDurationInSec = track.duration / track.timescale;
+        const segmentDurationInSec = edit.segmentDuration / track.timescale;
+        const segmentFrames =
+          track.nb_samples * (segmentDurationInSec / trackDurationInSec);
+        const mediaRate =
+          edit.mediaRateInteger + edit.mediaRateFraction / 0xffff;
+        const fps = (segmentFrames / segmentDurationInSec) * mediaRate;
+        return {...edit, fps};
+      });
+
+      file.setExtractionOptions(track.id);
+      file.start();
+
+      res({file, edits, config});
+    };
+
+    return fetch(uri).then(async response => {
+      if (!response.body) {
+        throw new Error('Response body is null');
+      }
+
+      const reader = response.body.getReader();
+      const sink = new MP4FileSink(file, () => {});
+
+      while (!found) {
+        await reader.read().then(({done, value}) => {
+          if (done) {
+            file.flush();
+            rej('Could not find moov');
+            return;
+          }
+
+          sink.write(value);
+        });
+      }
+    });
+  });
+}
+
+export class Mp4Parser {
+  private uri: string;
+  private file: any;
+  private edits: Edit[] = [];
+  private decoderConfig?: VideoDecoderConfig;
+
+  private startTime: number;
+  private targetFps: number;
+
+  private nextSegment = 0;
+  private sampler?: FrameSampler;
+
+  public constructor(uri: string, targetFps: number, startTime: number) {
+    this.uri = uri;
+    this.targetFps = targetFps;
+    this.startTime = startTime;
+  }
+
+  public async start() {
+    const {file, edits, config} = await getFileInfo(this.uri);
+    this.file = file;
+    this.edits = edits;
+    this.decoderConfig = config;
+  }
+
+  public async getNextFrame() {
+    // Start the first segment
+    if (!this.sampler) {
+      // Skip segments until the start time
+      let startTimeWithinSegment = this.startTime;
+      while (this.nextSegment < this.edits.length) {
+        const segmentDurationInSeconds = this.getSecondDurationOfSegment(
+          this.edits[this.nextSegment],
+        );
+        if (startTimeWithinSegment < segmentDurationInSeconds) {
+          break;
+        }
+
+        startTimeWithinSegment -= segmentDurationInSeconds;
+        this.nextSegment++;
+      }
+
+      // The timestamp is outside of the video
+      if (this.nextSegment >= this.edits.length) {
+        throw new Error(
+          `Timestamp ${this.startTime} is outside of the video, max timestamp is ${this.getDuration()}`,
+        );
+      }
+
+      const segment = new Segment(
+        this.uri,
+        this.file,
+        this.edits,
+        this.nextSegment,
+        startTimeWithinSegment,
+      );
+      await segment.start(this.decoderConfig!);
+
+      this.sampler = new FrameSampler(
+        segment,
+        this.edits[this.nextSegment].fps,
+        this.targetFps,
+        0,
+      );
+      this.nextSegment++;
+    }
+
+    // Try to get the next frame
+    let frame = await this.sampler.getNextFrame();
+
+    // If there are no more frames in the current segment, start the next segment
+    while (!frame && this.nextSegment < this.edits.length) {
+      this.sampler.close();
+      const segment = new Segment(
+        this.uri,
+        this.file,
+        this.edits,
+        this.nextSegment,
+        0,
+      );
+      await segment.start(this.decoderConfig!);
+
+      this.sampler = new FrameSampler(
+        segment,
+        this.edits[this.nextSegment].fps,
+        this.targetFps,
+        this.sampler.getSum(), // Carry over the sum from the previous segment
+      );
+      this.nextSegment++;
+
+      frame = await this.sampler.getNextFrame();
+    }
+
+    // If there are no more frames, return the last frame
+    if (!frame) {
+      frame = this.sampler.getLastFrame();
+
+      // If we still don't have a frame, throw an error
+      if (!frame) {
+        throw new Error('There are no frames in the video.');
+      }
+    }
+
+    return frame;
+  }
+
+  private getSecondDurationOfSegment(edit: Edit) {
+    const mediaRate = edit.mediaRateInteger + edit.mediaRateFraction / 0xffff;
+    const duration =
+      edit.segmentDuration / this.file.getInfo().videoTracks[0].timescale;
+    return duration / mediaRate;
+  }
+
+  public getDuration() {
+    return this.edits.reduce(
+      (acc, edit) => acc + this.getSecondDurationOfSegment(edit),
+      0,
+    );
+  }
+
+  private getTimeSubtractingFrames(frames: number) {
+    const durationOfPastSegmentsInSeconds = this.edits
+      .slice(0, this.nextSegment - 1)
+      .reduce((acc, edit) => acc + this.getSecondDurationOfSegment(edit), 0);
+
+    const currentSegment = this.sampler?.getSegment();
+    if (!currentSegment) {
+      throw new Error('No current segment');
+    }
+
+    const currentSegmentFrames = currentSegment.getFramesProcessed() - frames;
+    const currentSegmentFps = this.edits[this.nextSegment - 1].fps;
+    const currentSegmentDuration = currentSegmentFrames / currentSegmentFps;
+
+    return durationOfPastSegmentsInSeconds + currentSegmentDuration;
+  }
+
+  public getTime() {
+    return this.getTimeSubtractingFrames(0);
+  }
+
+  public getLastTime() {
+    return this.getTimeSubtractingFrames(1);
+  }
+
+  public getLastFrame() {
+    return this.sampler?.getLastFrame();
+  }
+
+  public close() {
+    this.sampler?.close();
+    this.file.close();
+  }
+}

package/src/lib/utils/video/parser/sampler.ts

@@ -0,0 +1,71 @@
+import {Segment} from './segment';
+
+export class FrameSampler {
+  private lastFrame?: VideoFrame;
+
+  public constructor(
+    private segment: Segment,
+    private sourceFps: number,
+    private targetFps: number,
+    private sum: number = 0,
+  ) {}
+
+  public async getNextFrame() {
+    const samplingRate = this.sourceFps / this.targetFps;
+    this.sum += samplingRate;
+
+    // Return last frame if we haven't progressed enough to get a new frame.
+    if (this.sum < 1 && this.lastFrame) {
+      return this.lastFrame;
+    }
+
+    // Burn frames if we have too many.
+    while (this.sum >= 2) {
+      // Burn frame
+      const frame = await this.segment.getNextFrame();
+
+      // If there are no more frames, return
+      if (!frame) {
+        return;
+      }
+
+      frame.close();
+      this.sum -= 1;
+    }
+
+    // Get a new frame.
+    if (this.sum >= 1 || !this.lastFrame) {
+      this.sum -= 1;
+      const frame = await this.segment.getNextFrame();
+
+      // If there are no more frames, return
+      if (!frame) {
+        return;
+      }
+
+      this.lastFrame?.close();
+      this.lastFrame = frame;
+      return frame;
+    }
+
+    // One of the three if statements above is always true.
+    throw new Error('Unreachable code');
+  }
+
+  public getLastFrame() {
+    return this.lastFrame;
+  }
+
+  public async close() {
+    this.lastFrame?.close();
+    this.segment.close();
+  }
+
+  public getSum() {
+    return this.sum;
+  }
+
+  public getSegment() {
+    return this.segment;
+  }
+}

package/src/lib/utils/video/parser/segment.ts

@@ -0,0 +1,197 @@
+import {MP4FileSink} from './sink';
+import {Edit} from './utils';
+
+export class Segment {
+  private done: boolean = false;
+  private uri: string;
+
+  private file: any;
+  private edit: Edit;
+
+  private responseFinished: boolean = false;
+  private decoder: VideoDecoder;
+
+  private framesProcessed = 0;
+  private startTime: number;
+  private framesDue = 0;
+  private frameBuffer: VideoFrame[] = [];
+
+  private readMore: () => Promise<void> = async () => {};
+
+  public constructor(
+    uri: string,
+    file: any,
+    edits: Edit[],
+    currentSegment: number,
+    startTime: number = 0,
+  ) {
+    this.uri = uri;
+
+    this.file = file;
+    this.file.onSamples = this.onSamples.bind(this);
+    this.edit = edits[currentSegment];
+
+    this.startTime = startTime;
+
+    // Check how many frames we skipped because of the startTime and add them to the frames processed.
+    const framesSkipped = Math.floor(this.startTime * this.edit.fps);
+    this.framesProcessed += framesSkipped;
+
+    this.decoder = new VideoDecoder({
+      output: this.onFrame.bind(this),
+      error(e) {
+        console.error(e);
+      },
+    });
+  }
+
+  public async start(decoderConfig: VideoDecoderConfig) {
+    this.decoder.configure(decoderConfig);
+    const videoTrack = this.file.getInfo().videoTracks[0];
+    const trak = this.file.getTrackById(videoTrack.id);
+
+    const mediaTimeInTimescale = this.edit.mediaTime;
+    const startTimeInTimescale = this.startTime * videoTrack.timescale;
+    const seekTimeInTimescale = mediaTimeInTimescale + startTimeInTimescale;
+
+    const seekTimeInSec = seekTimeInTimescale / videoTrack.timescale;
+    const seekInfo = this.file.seekTrack(seekTimeInSec, true, trak);
+    this.readMore = await this.startStreamingAtOffset(
+      this.file,
+      this.uri,
+      seekInfo.offset,
+    );
+  }
+
+  /**
+   * Starts streaming the video at the given URI from the given offset.
+   * @param file - MP4Box file. Needs to be created and configured before calling this function.
+   * @param uri - URI of the video file.
+   * @param offset - Offset to start streaming from.
+   * @returns - A function to read more data from the response.
+   */
+  private async startStreamingAtOffset(file: any, uri: string, offset: number) {
+    return fetch(uri, {
+      headers: {
+        /* eslint-disable-next-line @typescript-eslint/naming-convention */
+        Range: `bytes=${offset}-`,
+      },
+    }).then(async response => {
+      if (!response.body) {
+        throw new Error('Response body is null');
+      }
+
+      const reader = response.body.getReader();
+      const sink = new MP4FileSink(file, () => {}, offset);
+
+      return async () => {
+        return reader.read().then(({done, value}) => {
+          // Request is done.
+          if (done) {
+            this.responseFinished = true;
+            this.decoder.flush();
+            sink.close();
+            return;
+          }
+
+          sink.write(value);
+        });
+      };
+    });
+  }
+
+  /**
+   * Called when samples are available on the MP4 file.
+   * Sends chunks to the decoder.
+   */
+  private onSamples(_unused1: any, _unused2: any, samples: any) {
+    for (const sample of samples) {
+      const chunk = new EncodedVideoChunk({
+        type: sample.is_sync ? 'key' : 'delta',
+        timestamp: (1e6 * sample.cts) / sample.timescale,
+        duration: (1e6 * sample.duration) / sample.timescale,
+        data: sample.data,
+      });
+      this.framesDue++;
+      this.decoder.decode(chunk);
+    }
+  }
+
+  /**
+   * Called when the decoder has a frame ready.
+   * Pushes the frame to the buffer so it can be consumed.
+   */
+  private onFrame(frame: VideoFrame) {
+    this.framesDue--;
+
+    // If the frame comes before the seek time, close it.
+    const mediaTimeInSec =
+      this.edit.mediaTime / this.file.getInfo().videoTracks[0].timescale;
+    const seekTimeInSec = this.startTime + mediaTimeInSec;
+    const frameTimeInSec = frame.timestamp / 1e6;
+    if (frameTimeInSec < seekTimeInSec) {
+      frame.close();
+      return;
+    }
+
+    // Check if we are past the segment duration.
+    const segmentDurationInSec =
+      this.edit.segmentDuration / this.file.getInfo().videoTracks[0].timescale;
+    const segmentEndTime = mediaTimeInSec + segmentDurationInSec;
+    if (frameTimeInSec > segmentEndTime) {
+      frame.close();
+      this.done = true;
+      this.decoder.flush();
+      return;
+    }
+
+    this.frameBuffer.push(frame);
+  }
+
+  private async populateBuffer() {
+    // Fetch more frames if we don't have any.
+    while (this.frameBuffer.length === 0 && !this.responseFinished) {
+      await this.readMore();
+      await new Promise(res => setTimeout(res, 0));
+    }
+
+    // Wait for decoder if there are frames due.
+    if (this.frameBuffer.length === 0 && this.framesDue > 0) {
+      let maxIterations = 1000;
+      while (this.frameBuffer.length === 0) {
+        await new Promise(res => setTimeout(res, 10));
+        maxIterations--;
+
+        if (this.done) {
+          return;
+        }
+
+        if (maxIterations === 0) {
+          throw new Error(
+            `Timed out while waiting for VideoDecoder to produce a frame. Frames due: ${this.framesDue}`,
+          );
+        }
+      }
+    }
+  }
+
+  public async getNextFrame() {
+    await this.populateBuffer();
+    const frame = this.frameBuffer.shift();
+    if (frame) {
+      this.framesProcessed++;
+    }
+    return frame;
+  }
+
+  /**
+   * Called when we are done with the extractor.
+   */
+  public close() {
+    this.frameBuffer.forEach(frame => frame.close());
+  }
+
+  public getFramesProcessed() {
+    return this.framesProcessed;
+  }
+}

package/src/lib/utils/video/parser/sink.ts

@@ -0,0 +1,29 @@
+// Wraps an MP4Box File as a WritableStream underlying sink.
+export class MP4FileSink {
+  private setStatus;
+  private file;
+  private offset;
+
+  public constructor(file: any, setStatus: any, offset: number = 0) {
+    this.file = file;
+    this.setStatus = setStatus;
+    this.offset = offset;
+  }
+
+  public write(chunk: any) {
+    const buffer = new ArrayBuffer(chunk.byteLength);
+    new Uint8Array(buffer).set(chunk);
+
+    // Inform MP4Box where in the file this chunk is from.
+    (buffer as any).fileStart = this.offset;
+    this.offset += buffer.byteLength;
+
+    this.setStatus('fetch', (this.offset / 1024 ** 2).toFixed(1) + ' MiB');
+    this.file.appendBuffer(buffer);
+  }
+
+  public close() {
+    this.setStatus('fetch', 'Done');
+    this.file.flush();
+  }
+}

package/src/lib/utils/video/parser/utils.ts

@@ -0,0 +1,31 @@
+import {DataStream} from 'mp4box';
+
+export function description(file: any, track: any) {
+  const trak = file.getTrackById(track.id);
+  for (const entry of trak.mdia.minf.stbl.stsd.entries) {
+    const box = entry.avcC || entry.hvcC || entry.vpcC || entry.av1C;
+    if (box) {
+      const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
+      box.write(stream);
+      return new Uint8Array(stream.buffer, 8); // Remove the box header.
+    }
+  }
+  throw new Error('avcC, hvcC, vpcC, or av1C box not found');
+}
+
+export interface Edit {
+  mediaTime: number;
+  segmentDuration: number;
+  mediaRateInteger: number;
+  mediaRateFraction: number;
+  fps: number;
+}
+
+export function mp4BoxEditToEdit(edit: any): Omit<Edit, 'fps'> {
+  return {
+    mediaTime: edit.media_time,
+    segmentDuration: edit.segment_duration,
+    mediaRateInteger: edit.media_rate_integer,
+    mediaRateFraction: edit.media_rate_fraction,
+  };
+}