@revideo/2d 0.4.7-alpha.1024 → 0.4.7-alpha.1028

package/lib/utils/video/parser/segment.d.ts

@@ -0,0 +1,44 @@
+import { Edit } from './utils';
+export declare class Segment {
+    private done;
+    private abortController;
+    private uri;
+    private file;
+    private edit;
+    private responseFinished;
+    private decoder;
+    private framesProcessed;
+    private startTime;
+    private framesDue;
+    private frameBuffer;
+    private readMore;
+    constructor(uri: string, file: any, edits: Edit[], currentSegment: number, startTime?: number);
+    start(decoderConfig: VideoDecoderConfig): Promise<void>;
+    /**
+     * Starts streaming the video at the given URI from the given offset.
+     * @param file - MP4Box file. Needs to be created and configured before calling this function.
+     * @param uri - URI of the video file.
+     * @param offset - Offset to start streaming from.
+     * @returns - A function to read more data from the response.
+     */
+    private startStreamingAtOffset;
+    /**
+     * Called when samples are available on the MP4 file.
+     * Sends chunks to the decoder.
+     */
+    private onSamples;
+    /**
+     * Called when the decoder has a frame ready.
+     * Pushes the frame to the buffer so it can be consumed.
+     */
+    private onFrame;
+    private populateBuffer;
+    getNextFrame(): Promise<VideoFrame | undefined>;
+    /**
+     * Called when we are done with the extractor.
+     */
+    close(): void;
+    getFramesProcessed(): number;
+    getStartTime(): number;
+}
+//# sourceMappingURL=segment.d.ts.map

package/lib/utils/video/parser/segment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"segment.d.ts","sourceRoot":"","sources":["../../../../src/lib/utils/video/parser/segment.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,SAAS,CAAC;AAE7B,qBAAa,OAAO;IAClB,OAAO,CAAC,IAAI,CAAkB;IAC9B,OAAO,CAAC,eAAe,CAAyB;IAChD,OAAO,CAAC,GAAG,CAAS;IAEpB,OAAO,CAAC,IAAI,CAAM;IAClB,OAAO,CAAC,IAAI,CAAO;IAEnB,OAAO,CAAC,gBAAgB,CAAkB;IAC1C,OAAO,CAAC,OAAO,CAAe;IAE9B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,WAAW,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAuC;gBAGrD,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,GAAG,EACT,KAAK,EAAE,IAAI,EAAE,EACb,cAAc,EAAE,MAAM,EACtB,SAAS,GAAE,MAAU;IAsBV,KAAK,CAAC,aAAa,EAAE,kBAAkB;IAkBpD;;;;;;OAMG;YACW,sBAAsB;IAgCpC;;;OAGG;IACH,OAAO,CAAC,SAAS;IAiBjB;;;OAGG;IACH,OAAO,CAAC,OAAO;YA4BD,cAAc;IA2Bf,YAAY;IASzB;;OAEG;IACI,KAAK;IAML,kBAAkB;IAIlB,YAAY;CAGpB"}

package/lib/utils/video/parser/segment.js

@@ -0,0 +1,161 @@
+import { MP4FileSink } from './sink';
+export class Segment {
+    constructor(uri, file, edits, currentSegment, startTime = 0) {
+        this.done = false;
+        this.abortController = new AbortController();
+        this.responseFinished = false;
+        this.framesProcessed = 0;
+        this.framesDue = 0;
+        this.frameBuffer = [];
+        this.readMore = async () => { };
+        this.uri = uri;
+        this.file = file;
+        this.file.onSamples = this.onSamples.bind(this);
+        this.edit = edits[currentSegment];
+        this.startTime = startTime;
+        // Check how many frames we skipped because of the startTime and add them to the frames processed.
+        const framesSkipped = Math.floor(this.startTime * this.edit.fps);
+        this.framesProcessed += framesSkipped;
+        this.decoder = new VideoDecoder({
+            output: this.onFrame.bind(this),
+            error(e) {
+                console.error(e);
+            },
+        });
+    }
+    async start(decoderConfig) {
+        this.decoder.configure(decoderConfig);
+        const videoTrack = this.file.getInfo().videoTracks[0];
+        const trak = this.file.getTrackById(videoTrack.id);
+        const mediaTimeInTimescale = this.edit.mediaTime;
+        const startTimeInTimescale = this.startTime * videoTrack.timescale;
+        const seekTimeInTimescale = mediaTimeInTimescale + startTimeInTimescale;
+        const seekTimeInSec = seekTimeInTimescale / videoTrack.timescale;
+        const seekInfo = this.file.seekTrack(seekTimeInSec, true, trak);
+        this.readMore = await this.startStreamingAtOffset(this.file, this.uri, seekInfo.offset);
+    }
+    /**
+     * Starts streaming the video at the given URI from the given offset.
+     * @param file - MP4Box file. Needs to be created and configured before calling this function.
+     * @param uri - URI of the video file.
+     * @param offset - Offset to start streaming from.
+     * @returns - A function to read more data from the response.
+     */
+    async startStreamingAtOffset(file, uri, offset) {
+        return fetch(uri, {
+            headers: {
+                /* eslint-disable-next-line @typescript-eslint/naming-convention */
+                Range: `bytes=${offset}-`,
+            },
+            signal: this.abortController.signal,
+        }).then(async (response) => {
+            if (!response.body) {
+                throw new Error('Response body is null');
+            }
+            const reader = response.body.getReader();
+            const sink = new MP4FileSink(file, () => { }, offset);
+            return async () => {
+                return reader.read().then(({ done, value }) => {
+                    // Request is done.
+                    if (done) {
+                        this.responseFinished = true;
+                        this.abortController.abort();
+                        this.decoder.flush();
+                        sink.close();
+                        return;
+                    }
+                    sink.write(value);
+                });
+            };
+        });
+    }
+    /**
+     * Called when samples are available on the MP4 file.
+     * Sends chunks to the decoder.
+     */
+    onSamples(_unused1, _unused2, samples) {
+        for (const sample of samples) {
+            const chunk = new EncodedVideoChunk({
+                type: sample.is_sync ? 'key' : 'delta',
+                timestamp: (1e6 * sample.cts) / sample.timescale,
+                duration: (1e6 * sample.duration) / sample.timescale,
+                data: sample.data,
+            });
+            this.framesDue++;
+            this.decoder.decode(chunk);
+            const videoTrack = this.file.getInfo().videoTracks[0];
+            const trak = this.file.getTrackById(videoTrack.id);
+            this.file.releaseSample(trak, sample.number);
+        }
+    }
+    /**
+     * Called when the decoder has a frame ready.
+     * Pushes the frame to the buffer so it can be consumed.
+     */
+    onFrame(frame) {
+        this.framesDue--;
+        // If the frame comes before the seek time, close it.
+        const mediaTimeInSec = this.edit.mediaTime / this.file.getInfo().videoTracks[0].timescale;
+        const seekTimeInSec = this.startTime + mediaTimeInSec;
+        const frameTimeInSec = frame.timestamp / 1e6;
+        if (frameTimeInSec < seekTimeInSec) {
+            frame.close();
+            return;
+        }
+        // Check if we are past the segment duration.
+        const segmentDurationInSec = this.edit.segmentDuration /
+            this.file.getInfo().videoTracks[0].movie_timescale;
+        const segmentEndTime = mediaTimeInSec + segmentDurationInSec;
+        if (frameTimeInSec > segmentEndTime) {
+            frame.close();
+            this.done = true;
+            this.decoder.flush();
+            return;
+        }
+        this.frameBuffer.push(frame);
+    }
+    async populateBuffer() {
+        // Fetch more frames if we don't have any.
+        while (this.frameBuffer.length === 0 && !this.responseFinished) {
+            await this.readMore();
+            await new Promise(res => setTimeout(res, 0));
+        }
+        // Wait for decoder if there are frames due.
+        if (this.frameBuffer.length === 0 && this.framesDue > 0) {
+            let maxIterations = 1000;
+            while (this.frameBuffer.length === 0) {
+                await new Promise(res => setTimeout(res, 10));
+                maxIterations--;
+                if (this.done) {
+                    return;
+                }
+                if (maxIterations === 0) {
+                    throw new Error(`Timed out while waiting for VideoDecoder to produce a frame. Frames due: ${this.framesDue}`);
+                }
+            }
+        }
+    }
+    async getNextFrame() {
+        await this.populateBuffer();
+        const frame = this.frameBuffer.shift();
+        if (frame) {
+            this.framesProcessed++;
+        }
+        return frame;
+    }
+    /**
+     * Called when we are done with the extractor.
+     */
+    close() {
+        this.abortController.abort();
+        this.frameBuffer.forEach(frame => frame.close());
+        this.decoder.close();
+    }
+    getFramesProcessed() {
+        return this.framesProcessed;
+    }
+    getStartTime() {
+        return this.startTime;
+    }
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2VnbWVudC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uL3NyYy9saWIvdXRpbHMvdmlkZW8vcGFyc2VyL3NlZ21lbnQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFDLFdBQVcsRUFBQyxNQUFNLFFBQVEsQ0FBQztBQUduQyxNQUFNLE9BQU8sT0FBTztJQWtCbEIsWUFDRSxHQUFXLEVBQ1gsSUFBUyxFQUNULEtBQWEsRUFDYixjQUFzQixFQUN0QixZQUFvQixDQUFDO1FBdEJmLFNBQUksR0FBWSxLQUFLLENBQUM7UUFDdEIsb0JBQWUsR0FBRyxJQUFJLGVBQWUsRUFBRSxDQUFDO1FBTXhDLHFCQUFnQixHQUFZLEtBQUssQ0FBQztRQUdsQyxvQkFBZSxHQUFHLENBQUMsQ0FBQztRQUVwQixjQUFTLEdBQUcsQ0FBQyxDQUFDO1FBQ2QsZ0JBQVcsR0FBaUIsRUFBRSxDQUFDO1FBRS9CLGFBQVEsR0FBd0IsS0FBSyxJQUFJLEVBQUUsR0FBRSxDQUFDLENBQUM7UUFTckQsSUFBSSxDQUFDLEdBQUcsR0FBRyxHQUFHLENBQUM7UUFFZixJQUFJLENBQUMsSUFBSSxHQUFHLElBQUksQ0FBQztRQUNqQixJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsR0FBRyxJQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNoRCxJQUFJLENBQUMsSUFBSSxHQUFHLEtBQUssQ0FBQyxjQUFjLENBQUMsQ0FBQztRQUVsQyxJQUFJLENBQUMsU0FBUyxHQUFHLFNBQVMsQ0FBQztRQUUzQixrR0FBa0c7UUFDbEcsTUFBTSxhQUFhLEdBQUcsSUFBSSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsU0FBUyxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7UUFDakUsSUFBSSxDQUFDLGVBQWUsSUFBSSxhQUFhLENBQUM7UUFFdEMsSUFBSSxDQUFDLE9BQU8sR0FBRyxJQUFJLFlBQVksQ0FBQztZQUM5QixNQUFNLEVBQUUsSUFBSSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDO1lBQy9CLEtBQUssQ0FBQyxDQUFDO2dCQUNMLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUM7WUFDbkIsQ0FBQztTQUNGLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFTSxLQUFLLENBQUMsS0FBSyxDQUFDLGFBQWlDO1FBQ2xELElBQUksQ0FBQyxPQUFPLENBQUMsU0FBUyxDQUFDLGFBQWEsQ0FBQyxDQUFDO1FBQ3RDLE1BQU0sVUFBVSxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxDQUFDO1FBQ3RELE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLFVBQVUsQ0FBQyxFQUFFLENBQUMsQ0FBQztRQUVuRCxNQUFNLG9CQUFvQixHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDO1FBQ2pELE1BQU0sb0JBQW9CLEdBQUcsSUFBSSxDQUFDLFNBQVMsR0FBRyxVQUFVLENBQUMsU0FBUyxDQUFDO1FBQ25FLE1BQU0sbUJBQW1CLEdBQUcsb0JBQW9CLEdBQUcsb0JBQW9CLENBQUM7UUFFeEUsTUFBTSxhQUFhLEdBQUcsbUJBQW1CLEdBQUcsVUFBVSxDQUFDLFNBQVMsQ0FBQztRQUNqRSxNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxhQUFhLEVBQUUsSUFBSSxFQUFFLElBQUksQ0FBQyxDQUFDO1FBQ2hFLElBQUksQ0FBQyxRQUFRLEdBQUcsTUFBTSxJQUFJLENBQUMsc0JBQXNCLENBQy9DLElBQUksQ0FBQyxJQUFJLEVBQ1QsSUFBSSxDQUFDLEdBQUcsRUFDUixRQUFRLENBQUMsTUFBTSxDQUNoQixDQUFDO0lBQ0osQ0FBQztJQUVEOzs7Ozs7T0FNRztJQUNLLEtBQUssQ0FBQyxzQkFBc0IsQ0FBQyxJQUFTLEVBQUUsR0FBVyxFQUFFLE1BQWM7UUFDekUsT0FBTyxLQUFLLENBQUMsR0FBRyxFQUFFO1lBQ2hCLE9BQU8sRUFBRTtnQkFDUCxtRUFBbUU7Z0JBQ25FLEtBQUssRUFBRSxTQUFTLE1BQU0sR0FBRzthQUMxQjtZQUNELE1BQU0sRUFBRSxJQUFJLENBQUMsZUFBZSxDQUFDLE1BQU07U0FDcEMsQ0FBQyxDQUFDLElBQUksQ0FBQyxLQUFLLEVBQUMsUUFBUSxFQUFDLEVBQUU7WUFDdkIsSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLEVBQUUsQ0FBQztnQkFDbkIsTUFBTSxJQUFJLEtBQUssQ0FBQyx1QkFBdUIsQ0FBQyxDQUFDO1lBQzNDLENBQUM7WUFFRCxNQUFNLE1BQU0sR0FBRyxRQUFRLENBQUMsSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDO1lBQ3pDLE1BQU0sSUFBSSxHQUFHLElBQUksV0FBVyxDQUFDLElBQUksRUFBRSxHQUFHLEVBQUUsR0FBRSxDQUFDLEVBQUUsTUFBTSxDQUFDLENBQUM7WUFFckQsT0FBTyxLQUFLLElBQUksRUFBRTtnQkFDaEIsT0FBTyxNQUFNLENBQUMsSUFBSSxFQUFFLENBQUMsSUFBSSxDQUFDLENBQUMsRUFBQyxJQUFJLEVBQUUsS0FBSyxFQUFDLEVBQUUsRUFBRTtvQkFDMUMsbUJBQW1CO29CQUNuQixJQUFJLElBQUksRUFBRSxDQUFDO3dCQUNULElBQUksQ0FBQyxnQkFBZ0IsR0FBRyxJQUFJLENBQUM7d0JBQzdCLElBQUksQ0FBQyxlQUFlLENBQUMsS0FBSyxFQUFFLENBQUM7d0JBQzdCLElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUM7d0JBQ3JCLElBQUksQ0FBQyxLQUFLLEVBQUUsQ0FBQzt3QkFDYixPQUFPO29CQUNULENBQUM7b0JBRUQsSUFBSSxDQUFDLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQztnQkFDcEIsQ0FBQyxDQUFDLENBQUM7WUFDTCxDQUFDLENBQUM7UUFDSixDQUFDLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRDs7O09BR0c7SUFDSyxTQUFTLENBQUMsUUFBYSxFQUFFLFFBQWEsRUFBRSxPQUFZO1FBQzFELEtBQUssTUFBTSxNQUFNLElBQUksT0FBTyxFQUFFLENBQUM7WUFDN0IsTUFBTSxLQUFLLEdBQUcsSUFBSSxpQkFBaUIsQ0FBQztnQkFDbEMsSUFBSSxFQUFFLE1BQU0sQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsT0FBTztnQkFDdEMsU0FBUyxFQUFFLENBQUMsR0FBRyxHQUFHLE1BQU0sQ0FBQyxHQUFHLENBQUMsR0FBRyxNQUFNLENBQUMsU0FBUztnQkFDaEQsUUFBUSxFQUFFLENBQUMsR0FBRyxHQUFHLE1BQU0sQ0FBQyxRQUFRLENBQUMsR0FBRyxNQUFNLENBQUMsU0FBUztnQkFDcEQsSUFBSSxFQUFFLE1BQU0sQ0FBQyxJQUFJO2FBQ2xCLENBQUMsQ0FBQztZQUNILElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQztZQUNqQixJQUFJLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsQ0FBQztZQUUzQixNQUFNLFVBQVUsR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsQ0FBQztZQUN0RCxNQUFNLElBQUksR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLFlBQVksQ0FBQyxVQUFVLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDbkQsSUFBSSxDQUFDLElBQUksQ0FBQyxhQUFhLENBQUMsSUFBSSxFQUFFLE1BQU0sQ0FBQyxNQUFNLENBQUMsQ0FBQztRQUMvQyxDQUFDO0lBQ0gsQ0FBQztJQUVEOzs7T0FHRztJQUNLLE9BQU8sQ0FBQyxLQUFpQjtRQUMvQixJQUFJLENBQUMsU0FBUyxFQUFFLENBQUM7UUFFakIscURBQXFEO1FBQ3JELE1BQU0sY0FBYyxHQUNsQixJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxTQUFTLENBQUM7UUFDckUsTUFBTSxhQUFhLEdBQUcsSUFBSSxDQUFDLFNBQVMsR0FBRyxjQUFjLENBQUM7UUFDdEQsTUFBTSxjQUFjLEdBQUcsS0FBSyxDQUFDLFNBQVMsR0FBRyxHQUFHLENBQUM7UUFDN0MsSUFBSSxjQUFjLEdBQUcsYUFBYSxFQUFFLENBQUM7WUFDbkMsS0FBSyxDQUFDLEtBQUssRUFBRSxDQUFDO1lBQ2QsT0FBTztRQUNULENBQUM7UUFFRCw2Q0FBNkM7UUFDN0MsTUFBTSxvQkFBb0IsR0FDeEIsSUFBSSxDQUFDLElBQUksQ0FBQyxlQUFlO1lBQ3pCLElBQUksQ0FBQyxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxDQUFDLGVBQWUsQ0FBQztRQUNyRCxNQUFNLGNBQWMsR0FBRyxjQUFjLEdBQUcsb0JBQW9CLENBQUM7UUFDN0QsSUFBSSxjQUFjLEdBQUcsY0FBYyxFQUFFLENBQUM7WUFDcEMsS0FBSyxDQUFDLEtBQUssRUFBRSxDQUFDO1lBQ2QsSUFBSSxDQUFDLElBQUksR0FBRyxJQUFJLENBQUM7WUFDakIsSUFBSSxDQUFDLE9BQU8sQ0FBQyxLQUFLLEVBQUUsQ0FBQztZQUNyQixPQUFPO1FBQ1QsQ0FBQztRQUVELElBQUksQ0FBQyxXQUFXLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQy9CLENBQUM7SUFFTyxLQUFLLENBQUMsY0FBYztRQUMxQiwwQ0FBMEM7UUFDMUMsT0FBTyxJQUFJLENBQUMsV0FBVyxDQUFDLE1BQU0sS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsZ0JBQWdCLEVBQUUsQ0FBQztZQUMvRCxNQUFNLElBQUksQ0FBQyxRQUFRLEVBQUUsQ0FBQztZQUN0QixNQUFNLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsVUFBVSxDQUFDLEdBQUcsRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFDO1FBQy9DLENBQUM7UUFFRCw0Q0FBNEM7UUFDNUMsSUFBSSxJQUFJLENBQUMsV0FBVyxDQUFDLE1BQU0sS0FBSyxDQUFDLElBQUksSUFBSSxDQUFDLFNBQVMsR0FBRyxDQUFDLEVBQUUsQ0FBQztZQUN4RCxJQUFJLGFBQWEsR0FBRyxJQUFJLENBQUM7WUFDekIsT0FBTyxJQUFJLENBQUMsV0FBVyxDQUFDLE1BQU0sS0FBSyxDQUFDLEVBQUUsQ0FBQztnQkFDckMsTUFBTSxJQUFJLE9BQU8sQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLFVBQVUsQ0FBQyxHQUFHLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQztnQkFDOUMsYUFBYSxFQUFFLENBQUM7Z0JBRWhCLElBQUksSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO29CQUNkLE9BQU87Z0JBQ1QsQ0FBQztnQkFFRCxJQUFJLGFBQWEsS0FBSyxDQUFDLEVBQUUsQ0FBQztvQkFDeEIsTUFBTSxJQUFJLEtBQUssQ0FDYiw0RUFBNEUsSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUM3RixDQUFDO2dCQUNKLENBQUM7WUFDSCxDQUFDO1FBQ0gsQ0FBQztJQUNILENBQUM7SUFFTSxLQUFLLENBQUMsWUFBWTtRQUN2QixNQUFNLElBQUksQ0FBQyxjQUFjLEVBQUUsQ0FBQztRQUM1QixNQUFNLEtBQUssR0FBRyxJQUFJLENBQUMsV0FBVyxDQUFDLEtBQUssRUFBRSxDQUFDO1FBQ3ZDLElBQUksS0FBSyxFQUFFLENBQUM7WUFDVixJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7UUFDekIsQ0FBQztRQUNELE9BQU8sS0FBSyxDQUFDO0lBQ2YsQ0FBQztJQUVEOztPQUVHO0lBQ0ksS0FBSztRQUNWLElBQUksQ0FBQyxlQUFlLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDN0IsSUFBSSxDQUFDLFdBQVcsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQyxLQUFLLENBQUMsS0FBSyxFQUFFLENBQUMsQ0FBQztRQUNqRCxJQUFJLENBQUMsT0FBTyxDQUFDLEtBQUssRUFBRSxDQUFDO0lBQ3ZCLENBQUM7SUFFTSxrQkFBa0I7UUFDdkIsT0FBTyxJQUFJLENBQUMsZUFBZSxDQUFDO0lBQzlCLENBQUM7SUFFTSxZQUFZO1FBQ2pCLE9BQU8sSUFBSSxDQUFDLFNBQVMsQ0FBQztJQUN4QixDQUFDO0NBQ0YifQ==

package/lib/utils/video/parser/sink.d.ts

@@ -0,0 +1,9 @@
+export declare class MP4FileSink {
+    private setStatus;
+    private file;
+    private offset;
+    constructor(file: any, setStatus: any, offset?: number);
+    write(chunk: any): void;
+    close(): void;
+}
+//# sourceMappingURL=sink.d.ts.map

package/lib/utils/video/parser/sink.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sink.d.ts","sourceRoot":"","sources":["../../../../src/lib/utils/video/parser/sink.ts"],"names":[],"mappings":"AACA,qBAAa,WAAW;IACtB,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,IAAI,CAAC;IACb,OAAO,CAAC,MAAM,CAAC;gBAEI,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,GAAE,MAAU;IAMzD,KAAK,CAAC,KAAK,EAAE,GAAG;IAYhB,KAAK;CAIb"}

package/lib/utils/video/parser/sink.js

@@ -0,0 +1,22 @@
+// Wraps an MP4Box File as a WritableStream underlying sink.
+export class MP4FileSink {
+    constructor(file, setStatus, offset = 0) {
+        this.file = file;
+        this.setStatus = setStatus;
+        this.offset = offset;
+    }
+    write(chunk) {
+        const buffer = new ArrayBuffer(chunk.byteLength);
+        new Uint8Array(buffer).set(chunk);
+        // Inform MP4Box where in the file this chunk is from.
+        buffer.fileStart = this.offset;
+        this.offset += buffer.byteLength;
+        this.setStatus('fetch', (this.offset / 1024 ** 2).toFixed(1) + ' MiB');
+        this.file.appendBuffer(buffer);
+    }
+    close() {
+        this.setStatus('fetch', 'Done');
+        this.file.flush();
+    }
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2luay5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uL3NyYy9saWIvdXRpbHMvdmlkZW8vcGFyc2VyL3NpbmsudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsNERBQTREO0FBQzVELE1BQU0sT0FBTyxXQUFXO0lBS3RCLFlBQW1CLElBQVMsRUFBRSxTQUFjLEVBQUUsU0FBaUIsQ0FBQztRQUM5RCxJQUFJLENBQUMsSUFBSSxHQUFHLElBQUksQ0FBQztRQUNqQixJQUFJLENBQUMsU0FBUyxHQUFHLFNBQVMsQ0FBQztRQUMzQixJQUFJLENBQUMsTUFBTSxHQUFHLE1BQU0sQ0FBQztJQUN2QixDQUFDO0lBRU0sS0FBSyxDQUFDLEtBQVU7UUFDckIsTUFBTSxNQUFNLEdBQUcsSUFBSSxXQUFXLENBQUMsS0FBSyxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ2pELElBQUksVUFBVSxDQUFDLE1BQU0sQ0FBQyxDQUFDLEdBQUcsQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUVsQyxzREFBc0Q7UUFDckQsTUFBYyxDQUFDLFNBQVMsR0FBRyxJQUFJLENBQUMsTUFBTSxDQUFDO1FBQ3hDLElBQUksQ0FBQyxNQUFNLElBQUksTUFBTSxDQUFDLFVBQVUsQ0FBQztRQUVqQyxJQUFJLENBQUMsU0FBUyxDQUFDLE9BQU8sRUFBRSxDQUFDLElBQUksQ0FBQyxNQUFNLEdBQUcsSUFBSSxJQUFJLENBQUMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsR0FBRyxNQUFNLENBQUMsQ0FBQztRQUN2RSxJQUFJLENBQUMsSUFBSSxDQUFDLFlBQVksQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUNqQyxDQUFDO0lBRU0sS0FBSztRQUNWLElBQUksQ0FBQyxTQUFTLENBQUMsT0FBTyxFQUFFLE1BQU0sQ0FBQyxDQUFDO1FBQ2hDLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDcEIsQ0FBQztDQUNGIn0=

package/lib/utils/video/parser/utils.d.ts

@@ -0,0 +1,10 @@
+export declare function description(file: any, track: any): Uint8Array;
+export interface Edit {
+    mediaTime: number;
+    segmentDuration: number;
+    mediaRateInteger: number;
+    mediaRateFraction: number;
+    fps: number;
+}
+export declare function mp4BoxEditToEdit(edit: any): Omit<Edit, 'fps'>;
+//# sourceMappingURL=utils.d.ts.map

package/lib/utils/video/parser/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../../src/lib/utils/video/parser/utils.ts"],"names":[],"mappings":"AAEA,wBAAgB,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,cAWhD;AAED,MAAM,WAAW,IAAI;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,GAAG,EAAE,MAAM,CAAC;CACb;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAO7D"}

package/lib/utils/video/parser/utils.js

@@ -0,0 +1,22 @@
+import { DataStream } from 'mp4box';
+export function description(file, track) {
+    const trak = file.getTrackById(track.id);
+    for (const entry of trak.mdia.minf.stbl.stsd.entries) {
+        const box = entry.avcC || entry.hvcC || entry.vpcC || entry.av1C;
+        if (box) {
+            const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
+            box.write(stream);
+            return new Uint8Array(stream.buffer, 8); // Remove the box header.
+        }
+    }
+    throw new Error('avcC, hvcC, vpcC, or av1C box not found');
+}
+export function mp4BoxEditToEdit(edit) {
+    return {
+        mediaTime: edit.media_time,
+        segmentDuration: edit.segment_duration,
+        mediaRateInteger: edit.media_rate_integer,
+        mediaRateFraction: edit.media_rate_fraction,
+    };
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidXRpbHMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi9zcmMvbGliL3V0aWxzL3ZpZGVvL3BhcnNlci91dGlscy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUMsVUFBVSxFQUFDLE1BQU0sUUFBUSxDQUFDO0FBRWxDLE1BQU0sVUFBVSxXQUFXLENBQUMsSUFBUyxFQUFFLEtBQVU7SUFDL0MsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLFlBQVksQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDLENBQUM7SUFDekMsS0FBSyxNQUFNLEtBQUssSUFBSSxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ3JELE1BQU0sR0FBRyxHQUFHLEtBQUssQ0FBQyxJQUFJLElBQUksS0FBSyxDQUFDLElBQUksSUFBSSxLQUFLLENBQUMsSUFBSSxJQUFJLEtBQUssQ0FBQyxJQUFJLENBQUM7UUFDakUsSUFBSSxHQUFHLEVBQUUsQ0FBQztZQUNSLE1BQU0sTUFBTSxHQUFHLElBQUksVUFBVSxDQUFDLFNBQVMsRUFBRSxDQUFDLEVBQUUsVUFBVSxDQUFDLFVBQVUsQ0FBQyxDQUFDO1lBQ25FLEdBQUcsQ0FBQyxLQUFLLENBQUMsTUFBTSxDQUFDLENBQUM7WUFDbEIsT0FBTyxJQUFJLFVBQVUsQ0FBQyxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUMseUJBQXlCO1FBQ3BFLENBQUM7SUFDSCxDQUFDO0lBQ0QsTUFBTSxJQUFJLEtBQUssQ0FBQyx5Q0FBeUMsQ0FBQyxDQUFDO0FBQzdELENBQUM7QUFVRCxNQUFNLFVBQVUsZ0JBQWdCLENBQUMsSUFBUztJQUN4QyxPQUFPO1FBQ0wsU0FBUyxFQUFFLElBQUksQ0FBQyxVQUFVO1FBQzFCLGVBQWUsRUFBRSxJQUFJLENBQUMsZ0JBQWdCO1FBQ3RDLGdCQUFnQixFQUFFLElBQUksQ0FBQyxrQkFBa0I7UUFDekMsaUJBQWlCLEVBQUUsSUFBSSxDQUFDLG1CQUFtQjtLQUM1QyxDQUFDO0FBQ0osQ0FBQyJ9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@revideo/2d",
-  "version": "0.4.7-alpha.1024+6b376d6f",
+  "version": "0.4.7-alpha.1028+5c7ad7b5",
   "description": "A 2D renderer for revideo",
   "author": "revideo",
   "homepage": "https://re.video/",
@@ -30,7 +30,7 @@
   "devDependencies": {
     "@preact/signals": "^1.2.1",
     "@revideo/internal": "^0.0.0",
-    "@revideo/ui": "^0.4.7-alpha.1024+6b376d6f",
+    "@revideo/ui": "^0.4.7-alpha.1028+5c7ad7b5",
     "clsx": "^2.0.0",
     "jsdom": "^22.1.0",
     "preact": "^10.19.2",
@@ -40,12 +40,12 @@
     "@codemirror/language": "^6.10.1",
     "@lezer/common": "^1.2.1",
     "@lezer/highlight": "^1.2.0",
-    "@revideo/core": "^0.4.7-alpha.1024+6b376d6f",
+    "@revideo/core": "^0.4.7-alpha.1028+5c7ad7b5",
     "code-fns": "^0.8.2",
     "hls.js": "^1.5.11",
     "mathjax-full": "^3.2.2",
     "mp4box": "^0.5.2",
     "parse-svg-path": "^0.1.2"
   },
-  "gitHead": "6b376d6fd166f21e381296f17f0a8f89e2db83ca"
+  "gitHead": "5c7ad7b52c0aa2c94186dc198ab0fdb989937803"
 }

package/src/lib/components/Video.ts

@@ -12,7 +12,7 @@ import {computed, initial, nodeName, signal} from '../decorators';
 import {DesiredLength} from '../partials';
 import {drawImage} from '../utils';
 import {ImageCommunication} from '../utils/video/ffmpeg-client';
-import {getFrame} from '../utils/video/mp4-parser-manager';
+import {dropExtractor, getFrame} from '../utils/video/mp4-parser-manager';
 import {Media, MediaProps} from './Media';
 
 export interface VideoProps extends MediaProps {
@@ -326,6 +326,12 @@ export class Video extends Media {
     ).toString();
   }
 
+  public override remove() {
+    super.remove();
+    dropExtractor(this.key, this.video().src);
+    return this;
+  }
+
   private async detectFileType(): Promise<
     'mp4' | 'webm' | 'hls' | 'mov' | 'unknown'
   > {

package/src/lib/utils/video/mp4-parser-manager.ts

@@ -1,7 +1,16 @@
-import {FrameExtractor} from './mp4-parser';
+import {Mp4Parser} from './parser';
 
 // List of VideoFrameExtractors
-const videoFrameExtractors = new Map<string, FrameExtractor>();
+const videoFrameExtractors = new Map<string, Mp4Parser>();
+
+export async function dropExtractor(id: string, filePath: string) {
+  const extractorId = filePath + '-' + id;
+  const extractor = videoFrameExtractors.get(extractorId);
+  if (extractor) {
+    extractor.close();
+    videoFrameExtractors.delete(extractorId);
+  }
+}
 
 export async function getFrame(
   id: string,
@@ -15,12 +24,22 @@ export async function getFrame(
 
   const frameDuration = 1 / fps;
 
+  /**
+   * Sometimes, HTMLVideoElement.duration is not accurate, which can lead to the
+   * requested time being greater than the duration.
+   * To prevent this, we clamp the time to the duration reported by the extractor.
+   */
+  const duration = extractor?.getDuration();
+  if (duration && time > duration) {
+    time = duration;
+  }
+
   const isOldFrame =
     extractor && Math.abs(time - extractor.getLastTime()) < frameDuration / 2;
 
   // If time has not changed, return the last frame
-  if (isOldFrame) {
-    const lastFrame = extractor!.getLastFrame();
+  if (extractor && isOldFrame) {
+    const lastFrame = extractor.getLastFrame();
     if (!lastFrame) {
       throw new Error('No last frame');
     }
@@ -43,7 +62,7 @@ export async function getFrame(
   }
 
   if (!extractor) {
-    extractor = new FrameExtractor(filePath, fps, time);
+    extractor = new Mp4Parser(filePath, fps, time);
     await extractor.start();
     videoFrameExtractors.set(extractorId, extractor);
   }

package/src/lib/utils/video/parser/index.ts

@@ -0,0 +1 @@
+export * from './parser';

package/src/lib/utils/video/parser/parser.ts

@@ -0,0 +1,256 @@
+import {createFile} from 'mp4box';
+import {FrameSampler} from './sampler';
+import {Segment} from './segment';
+import {MP4FileSink} from './sink';
+import {Edit, description, mp4BoxEditToEdit} from './utils';
+
+/**
+ * Loads the file at the given URI until it finds the moov box.
+ * Once found, it calls `setConfig` with the video configuration.
+ * @param uri - The URI of the video file.
+ * @param setConfig - Callback to set the video configuration.
+ * @returns
+ */
+async function getFileInfo(uri: string) {
+  return new Promise<{
+    file: any;
+    edits: Edit[];
+    config: VideoDecoderConfig;
+  }>((res, rej) => {
+    const file = createFile();
+    let found = false;
+    const controller = new AbortController();
+
+    file.onReady = (info: any) => {
+      found = true;
+      controller.abort();
+
+      const track = info.videoTracks[0];
+
+      const config = {
+        // Browser doesn't support parsing full vp8 codec (eg: `vp08.00.41.08`),
+        // they only support `vp8`.
+        codec: track.codec.startsWith('vp08') ? 'vp8' : track.codec,
+        codedHeight: track.video.height,
+        codedWidth: track.video.width,
+        description: description(file, track),
+      };
+
+      const defaultEdit: Omit<Edit, 'fps'> = {
+        mediaTime: 0,
+        segmentDuration: track.duration,
+        mediaRateInteger: 1,
+        mediaRateFraction: 0,
+      };
+
+      const editsFromVideo: Omit<Edit, 'fps'>[] = track.edits?.map(
+        (edit: any) => mp4BoxEditToEdit(edit),
+      );
+
+      // If there are no entries, use the default edit
+      const editsWithoutFps = editsFromVideo?.length
+        ? editsFromVideo
+        : [defaultEdit];
+
+      // Calculate FPS for each segment
+      const edits = editsWithoutFps.map(edit => {
+        const trackDurationInSec = track.duration / track.timescale;
+        const segmentDurationInSec =
+          edit.segmentDuration / track.movie_timescale;
+        const segmentFrames =
+          track.nb_samples * (segmentDurationInSec / trackDurationInSec);
+        const mediaRate =
+          edit.mediaRateInteger + edit.mediaRateFraction / 0xffff;
+        const fps = (segmentFrames / segmentDurationInSec) * mediaRate;
+        return {...edit, fps};
+      });
+
+      file.setExtractionOptions(track.id);
+      file.start();
+
+      res({file, edits, config});
+    };
+
+    return fetch(uri, {signal: controller.signal}).then(async response => {
+      if (!response.body) {
+        throw new Error('Response body is null');
+      }
+
+      const reader = response.body.getReader();
+      const sink = new MP4FileSink(file, () => {});
+
+      while (!found) {
+        await reader.read().then(({done, value}) => {
+          if (done) {
+            file.flush();
+            controller.abort();
+            rej('Could not find moov');
+            return;
+          }
+
+          sink.write(value);
+        });
+      }
+    });
+  });
+}
+
+export class Mp4Parser {
+  private uri: string;
+  private file: any;
+  private edits: Edit[] = [];
+  private decoderConfig?: VideoDecoderConfig;
+
+  private startTime: number;
+  private targetFps: number;
+
+  private nextSegment = 0;
+  private sampler?: FrameSampler;
+
+  public constructor(uri: string, targetFps: number, startTime: number) {
+    this.uri = uri;
+    this.targetFps = targetFps;
+    this.startTime = startTime;
+  }
+
+  public async start() {
+    const {file, edits, config} = await getFileInfo(this.uri);
+    this.file = file;
+    this.edits = edits;
+    this.decoderConfig = config;
+  }
+
+  public async getNextFrame() {
+    // Start the first segment
+    if (!this.sampler) {
+      // Skip segments until the start time
+      let startTimeWithinSegment = this.startTime;
+      while (this.nextSegment < this.edits.length) {
+        const segmentDurationInSeconds = this.getSecondDurationOfSegment(
+          this.edits[this.nextSegment],
+        );
+        if (startTimeWithinSegment < segmentDurationInSeconds) {
+          break;
+        }
+
+        startTimeWithinSegment -= segmentDurationInSeconds;
+        this.nextSegment++;
+      }
+
+      // The timestamp is outside of the video
+      if (this.nextSegment >= this.edits.length) {
+        throw new Error(
+          `Timestamp ${this.startTime} is outside of the video, max timestamp is ${this.getDuration()}`,
+        );
+      }
+
+      const segment = new Segment(
+        this.uri,
+        this.file,
+        this.edits,
+        this.nextSegment,
+        startTimeWithinSegment,
+      );
+      await segment.start(this.decoderConfig!);
+
+      this.sampler = new FrameSampler(
+        segment,
+        this.edits[this.nextSegment].fps,
+        this.targetFps,
+        0,
+      );
+      this.nextSegment++;
+    }
+
+    // Try to get the next frame
+    let frame = await this.sampler.getNextFrame();
+
+    // If there are no more frames in the current segment, start the next segment
+    while (!frame && this.nextSegment < this.edits.length) {
+      this.sampler.getSegment().close();
+      this.sampler.getLastFrame()?.close();
+      const segment = new Segment(
+        this.uri,
+        this.file,
+        this.edits,
+        this.nextSegment,
+        0,
+      );
+      await segment.start(this.decoderConfig!);
+
+      this.sampler = new FrameSampler(
+        segment,
+        this.edits[this.nextSegment].fps,
+        this.targetFps,
+        this.sampler.getSum(), // Carry over the sum from the previous segment
+      );
+      this.nextSegment++;
+
+      frame = await this.sampler.getNextFrame();
+    }
+
+    // If there are no more frames, return the last frame
+    if (!frame) {
+      frame = this.sampler.getLastFrame();
+
+      // We can close the segment
+      this.sampler.getSegment().close();
+
+      // If we still don't have a frame, throw an error
+      if (!frame) {
+        throw new Error('There are no frames in the video.');
+      }
+    }
+
+    return frame;
+  }
+
+  private getSecondDurationOfSegment(edit: Edit) {
+    const mediaRate = edit.mediaRateInteger + edit.mediaRateFraction / 0xffff;
+    const duration =
+      edit.segmentDuration / this.file.getInfo().videoTracks[0].movie_timescale;
+    return duration / mediaRate;
+  }
+
+  public getDuration() {
+    return this.edits.reduce(
+      (acc, edit) => acc + this.getSecondDurationOfSegment(edit),
+      0,
+    );
+  }
+
+  private getTimeSubtractingFrames(frames: number) {
+    const durationOfPastSegmentsInSeconds = this.edits
+      .slice(0, this.nextSegment - 1)
+      .reduce((acc, edit) => acc + this.getSecondDurationOfSegment(edit), 0);
+
+    if (!this.sampler) {
+      throw new Error('No current segment');
+    }
+
+    const currentSegmentStartTime = this.sampler.getSegment().getStartTime();
+    const samplerTime = this.sampler.getTime(frames);
+
+    return (
+      durationOfPastSegmentsInSeconds + currentSegmentStartTime + samplerTime
+    );
+  }
+
+  public getTime() {
+    return this.getTimeSubtractingFrames(0);
+  }
+
+  public getLastTime() {
+    return this.getTimeSubtractingFrames(1);
+  }
+
+  public getLastFrame() {
+    return this.sampler?.getLastFrame();
+  }
+
+  public close() {
+    this.sampler?.getSegment().close();
+    this.sampler?.getLastFrame()?.close();
+    this.file.flush();
+  }
+}