@revideo/2d 0.4.7-two.1024 → 0.4.8-alpha.1032

package/lib/utils/video/parser/segment.d.ts

@@ -0,0 +1,44 @@
+import { Edit } from './utils';
+export declare class Segment {
+    private done;
+    private abortController;
+    private uri;
+    private file;
+    private edit;
+    private responseFinished;
+    private decoder;
+    private framesProcessed;
+    private startTime;
+    private framesDue;
+    private frameBuffer;
+    private readMore;
+    constructor(uri: string, file: any, edits: Edit[], currentSegment: number, startTime?: number);
+    start(decoderConfig: VideoDecoderConfig): Promise<void>;
+    /**
+     * Starts streaming the video at the given URI from the given offset.
+     * @param file - MP4Box file. Needs to be created and configured before calling this function.
+     * @param uri - URI of the video file.
+     * @param offset - Offset to start streaming from.
+     * @returns - A function to read more data from the response.
+     */
+    private startStreamingAtOffset;
+    /**
+     * Called when samples are available on the MP4 file.
+     * Sends chunks to the decoder.
+     */
+    private onSamples;
+    /**
+     * Called when the decoder has a frame ready.
+     * Pushes the frame to the buffer so it can be consumed.
+     */
+    private onFrame;
+    private populateBuffer;
+    getNextFrame(): Promise<VideoFrame | undefined>;
+    /**
+     * Called when we are done with the extractor.
+     */
+    close(): Promise<void>;
+    getFramesProcessed(): number;
+    getStartTime(): number;
+}
+//# sourceMappingURL=segment.d.ts.map

package/lib/utils/video/parser/segment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"segment.d.ts","sourceRoot":"","sources":["../../../../src/lib/utils/video/parser/segment.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,SAAS,CAAC;AAE7B,qBAAa,OAAO;IAClB,OAAO,CAAC,IAAI,CAAkB;IAC9B,OAAO,CAAC,eAAe,CAAyB;IAChD,OAAO,CAAC,GAAG,CAAS;IAEpB,OAAO,CAAC,IAAI,CAAM;IAClB,OAAO,CAAC,IAAI,CAAO;IAEnB,OAAO,CAAC,gBAAgB,CAAkB;IAC1C,OAAO,CAAC,OAAO,CAAe;IAE9B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,WAAW,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAuC;gBAGrD,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,GAAG,EACT,KAAK,EAAE,IAAI,EAAE,EACb,cAAc,EAAE,MAAM,EACtB,SAAS,GAAE,MAAU;IAsBV,KAAK,CAAC,aAAa,EAAE,kBAAkB;IAoDpD;;;;;;OAMG;YACW,sBAAsB;IAgCpC;;;OAGG;IACH,OAAO,CAAC,SAAS;IAiBjB;;;OAGG;IACH,OAAO,CAAC,OAAO;YA4BD,cAAc;IA2Bf,YAAY;IASzB;;OAEG;IACU,KAAK;IAaX,kBAAkB;IAIlB,YAAY;CAGpB"}

package/lib/utils/video/parser/segment.js

@@ -0,0 +1,195 @@
+import { MP4FileSink } from './sink';
+export class Segment {
+    constructor(uri, file, edits, currentSegment, startTime = 0) {
+        this.done = false;
+        this.abortController = new AbortController();
+        this.responseFinished = false;
+        this.framesProcessed = 0;
+        this.framesDue = 0;
+        this.frameBuffer = [];
+        this.readMore = async () => { };
+        this.uri = uri;
+        this.file = file;
+        this.file.onSamples = this.onSamples.bind(this);
+        this.edit = edits[currentSegment];
+        this.startTime = startTime;
+        // Check how many frames we skipped because of the startTime and add them to the frames processed.
+        const framesSkipped = Math.floor(this.startTime * this.edit.fps);
+        this.framesProcessed += framesSkipped;
+        this.decoder = new VideoDecoder({
+            output: this.onFrame.bind(this),
+            error(e) {
+                console.error(e);
+            },
+        });
+    }
+    async start(decoderConfig) {
+        // If this is an empty edit, we just need to fill the buffer with empty frames.
+        if (this.edit.mediaTime === -1) {
+            this.done = true;
+            this.responseFinished = true;
+            this.decoder.close();
+            const segmentDurationInSeconds = this.edit.segmentDuration /
+                this.file.getInfo().videoTracks[0].movie_timescale;
+            const framesToFill = segmentDurationInSeconds * this.edit.fps;
+            const height = this.file.getInfo().videoTracks[0].track_height;
+            const width = this.file.getInfo().videoTracks[0].track_width;
+            const bufferSize = height * width * 4;
+            const buffer = new ArrayBuffer(bufferSize);
+            const uint8Array = new Uint8Array(buffer);
+            // Make the frame black
+            uint8Array.fill(0);
+            this.frameBuffer = Array.from({ length: framesToFill }, () => {
+                return new VideoFrame(uint8Array, {
+                    timestamp: 0,
+                    duration: 1 / this.edit.fps,
+                    codedHeight: height,
+                    codedWidth: width,
+                    format: 'BGRA',
+                });
+            });
+            return;
+        }
+        this.decoder.configure(decoderConfig);
+        const videoTrack = this.file.getInfo().videoTracks[0];
+        const trak = this.file.getTrackById(videoTrack.id);
+        const mediaTimeInTimescale = this.edit.mediaTime;
+        const startTimeInTimescale = this.startTime * videoTrack.timescale;
+        const seekTimeInTimescale = mediaTimeInTimescale + startTimeInTimescale;
+        const seekTimeInSec = seekTimeInTimescale / videoTrack.timescale;
+        const seekInfo = this.file.seekTrack(seekTimeInSec, true, trak);
+        this.readMore = await this.startStreamingAtOffset(this.file, this.uri, seekInfo.offset);
+    }
+    /**
+     * Starts streaming the video at the given URI from the given offset.
+     * @param file - MP4Box file. Needs to be created and configured before calling this function.
+     * @param uri - URI of the video file.
+     * @param offset - Offset to start streaming from.
+     * @returns - A function to read more data from the response.
+     */
+    async startStreamingAtOffset(file, uri, offset) {
+        return fetch(uri, {
+            headers: {
+                /* eslint-disable-next-line @typescript-eslint/naming-convention */
+                Range: `bytes=${offset}-`,
+            },
+            signal: this.abortController.signal,
+        }).then(async (response) => {
+            if (!response.body) {
+                throw new Error('Response body is null');
+            }
+            const reader = response.body.getReader();
+            const sink = new MP4FileSink(file, () => { }, offset);
+            return async () => {
+                return reader.read().then(({ done, value }) => {
+                    // Request is done.
+                    if (done) {
+                        this.responseFinished = true;
+                        this.abortController.abort();
+                        this.decoder.flush();
+                        sink.close();
+                        return;
+                    }
+                    sink.write(value);
+                });
+            };
+        });
+    }
+    /**
+     * Called when samples are available on the MP4 file.
+     * Sends chunks to the decoder.
+     */
+    onSamples(_unused1, _unused2, samples) {
+        for (const sample of samples) {
+            const chunk = new EncodedVideoChunk({
+                type: sample.is_sync ? 'key' : 'delta',
+                timestamp: (1e6 * sample.cts) / sample.timescale,
+                duration: (1e6 * sample.duration) / sample.timescale,
+                data: sample.data,
+            });
+            this.framesDue++;
+            this.decoder.decode(chunk);
+            const videoTrack = this.file.getInfo().videoTracks[0];
+            const trak = this.file.getTrackById(videoTrack.id);
+            this.file.releaseSample(trak, sample.number);
+        }
+    }
+    /**
+     * Called when the decoder has a frame ready.
+     * Pushes the frame to the buffer so it can be consumed.
+     */
+    onFrame(frame) {
+        this.framesDue--;
+        // If the frame comes before the seek time, close it.
+        const mediaTimeInSec = this.edit.mediaTime / this.file.getInfo().videoTracks[0].timescale;
+        const seekTimeInSec = this.startTime + mediaTimeInSec;
+        const frameTimeInSec = frame.timestamp / 1e6;
+        if (frameTimeInSec < seekTimeInSec) {
+            frame.close();
+            return;
+        }
+        // Check if we are past the segment duration.
+        const segmentDurationInSec = this.edit.segmentDuration /
+            this.file.getInfo().videoTracks[0].movie_timescale;
+        const segmentEndTime = mediaTimeInSec + segmentDurationInSec;
+        if (frameTimeInSec > segmentEndTime) {
+            frame.close();
+            this.done = true;
+            this.decoder.flush();
+            return;
+        }
+        this.frameBuffer.push(frame);
+    }
+    async populateBuffer() {
+        // Fetch more frames if we don't have any.
+        while (this.frameBuffer.length === 0 && !this.responseFinished) {
+            await this.readMore();
+            await new Promise(res => setTimeout(res, 0));
+        }
+        // Wait for decoder if there are frames due.
+        if (this.frameBuffer.length === 0 && this.framesDue > 0) {
+            let maxIterations = 1000;
+            while (this.frameBuffer.length === 0) {
+                await new Promise(res => setTimeout(res, 10));
+                maxIterations--;
+                if (this.done) {
+                    return;
+                }
+                if (maxIterations === 0) {
+                    throw new Error(`Timed out while waiting for VideoDecoder to produce a frame. Frames due: ${this.framesDue}`);
+                }
+            }
+        }
+    }
+    async getNextFrame() {
+        await this.populateBuffer();
+        const frame = this.frameBuffer.shift();
+        if (frame) {
+            this.framesProcessed++;
+        }
+        return frame;
+    }
+    /**
+     * Called when we are done with the extractor.
+     */
+    async close() {
+        this.abortController.abort();
+        this.frameBuffer.forEach(frame => frame.close());
+        try {
+            if (this.decoder.state === 'configured') {
+                await this.decoder.flush();
+                this.decoder.close();
+            }
+        }
+        catch (e) {
+            // Ignore
+        }
+    }
+    getFramesProcessed() {
+        return this.framesProcessed;
+    }
+    getStartTime() {
+        return this.startTime;
+    }
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2VnbWVudC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uL3NyYy9saWIvdXRpbHMvdmlkZW8vcGFyc2VyL3NlZ21lbnQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFDLFdBQVcsRUFBQyxNQUFNLFFBQVEsQ0FBQztBQUduQyxNQUFNLE9BQU8sT0FBTztJQWtCbEIsWUFDRSxHQUFXLEVBQ1gsSUFBUyxFQUNULEtBQWEsRUFDYixjQUFzQixFQUN0QixZQUFvQixDQUFDO1FBdEJmLFNBQUksR0FBWSxLQUFLLENBQUM7UUFDdEIsb0JBQWUsR0FBRyxJQUFJLGVBQWUsRUFBRSxDQUFDO1FBTXhDLHFCQUFnQixHQUFZLEtBQUssQ0FBQztRQUdsQyxvQkFBZSxHQUFHLENBQUMsQ0FBQztRQUVwQixjQUFTLEdBQUcsQ0FBQyxDQUFDO1FBQ2QsZ0JBQVcsR0FBaUIsRUFBRSxDQUFDO1FBRS9CLGFBQVEsR0FBd0IsS0FBSyxJQUFJLEVBQUUsR0FBRSxDQUFDLENBQUM7UUFTckQsSUFBSSxDQUFDLEdBQUcsR0FBRyxHQUFHLENBQUM7UUFFZixJQUFJLENBQUMsSUFBSSxHQUFHLElBQUksQ0FBQztRQUNqQixJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsR0FBRyxJQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNoRCxJQUFJLENBQUMsSUFBSSxHQUFHLEtBQUssQ0FBQyxjQUFjLENBQUMsQ0FBQztRQUVsQyxJQUFJLENBQUMsU0FBUyxHQUFHLFNBQVMsQ0FBQztRQUUzQixrR0FBa0c7UUFDbEcsTUFBTSxhQUFhLEdBQUcsSUFBSSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsU0FBUyxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7UUFDakUsSUFBSSxDQUFDLGVBQWUsSUFBSSxhQUFhLENBQUM7UUFFdEMsSUFBSSxDQUFDLE9BQU8sR0FBRyxJQUFJLFlBQVksQ0FBQztZQUM5QixNQUFNLEVBQUUsSUFBSSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDO1lBQy9CLEtBQUssQ0FBQyxDQUFDO2dCQUNMLE9BQU8sQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUM7WUFDbkIsQ0FBQztTQUNGLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFTSxLQUFLLENBQUMsS0FBSyxDQUFDLGFBQWlDO1FBQ2xELCtFQUErRTtRQUMvRSxJQUFJLElBQUksQ0FBQyxJQUFJLENBQUMsU0FBUyxLQUFLLENBQUMsQ0FBQyxFQUFFLENBQUM7WUFDL0IsSUFBSSxDQUFDLElBQUksR0FBRyxJQUFJLENBQUM7WUFDakIsSUFBSSxDQUFDLGdCQUFnQixHQUFHLElBQUksQ0FBQztZQUM3QixJQUFJLENBQUMsT0FBTyxDQUFDLEtBQUssRUFBRSxDQUFDO1lBRXJCLE1BQU0sd0JBQXdCLEdBQzVCLElBQUksQ0FBQyxJQUFJLENBQUMsZUFBZTtnQkFDekIsSUFBSSxDQUFDLElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLENBQUMsZUFBZSxDQUFDO1lBQ3JELE1BQU0sWUFBWSxHQUFHLHdCQUF3QixHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDO1lBRTlELE1BQU0sTUFBTSxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUMsV0FBVyxDQUFDLENBQUMsQ0FBQyxDQUFDLFlBQVksQ0FBQztZQUMvRCxNQUFNLEtBQUssR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxXQUFXLENBQUM7WUFFN0QsTUFBTSxVQUFVLEdBQUcsTUFBTSxHQUFHLEtBQUssR0FBRyxDQUFDLENBQUM7WUFDdEMsTUFBTSxNQUFNLEdBQUcsSUFBSSxXQUFXLENBQUMsVUFBVSxDQUFDLENBQUM7WUFDM0MsTUFBTSxVQUFVLEdBQUcsSUFBSSxVQUFVLENBQUMsTUFBTSxDQUFDLENBQUM7WUFFMUMsdUJBQXVCO1lBQ3ZCLFVBQVUsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUM7WUFFbkIsSUFBSSxDQUFDLFdBQVcsR0FBRyxLQUFLLENBQUMsSUFBSSxDQUFDLEVBQUMsTUFBTSxFQUFFLFlBQVksRUFBQyxFQUFFLEdBQUcsRUFBRTtnQkFDekQsT0FBTyxJQUFJLFVBQVUsQ0FBQyxVQUFVLEVBQUU7b0JBQ2hDLFNBQVMsRUFBRSxDQUFDO29CQUNaLFFBQVEsRUFBRSxDQUFDLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxHQUFHO29CQUMzQixXQUFXLEVBQUUsTUFBTTtvQkFDbkIsVUFBVSxFQUFFLEtBQUs7b0JBQ2pCLE1BQU0sRUFBRSxNQUFNO2lCQUNmLENBQUMsQ0FBQztZQUNMLENBQUMsQ0FBQyxDQUFDO1lBRUgsT0FBTztRQUNULENBQUM7UUFFRCxJQUFJLENBQUMsT0FBTyxDQUFDLFNBQVMsQ0FBQyxhQUFhLENBQUMsQ0FBQztRQUN0QyxNQUFNLFVBQVUsR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsQ0FBQztRQUN0RCxNQUFNLElBQUksR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLFlBQVksQ0FBQyxVQUFVLENBQUMsRUFBRSxDQUFDLENBQUM7UUFFbkQsTUFBTSxvQkFBb0IsR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQztRQUNqRCxNQUFNLG9CQUFvQixHQUFHLElBQUksQ0FBQyxTQUFTLEdBQUcsVUFBVSxDQUFDLFNBQVMsQ0FBQztRQUNuRSxNQUFNLG1CQUFtQixHQUFHLG9CQUFvQixHQUFHLG9CQUFvQixDQUFDO1FBRXhFLE1BQU0sYUFBYSxHQUFHLG1CQUFtQixHQUFHLFVBQVUsQ0FBQyxTQUFTLENBQUM7UUFDakUsTUFBTSxRQUFRLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsYUFBYSxFQUFFLElBQUksRUFBRSxJQUFJLENBQUMsQ0FBQztRQUNoRSxJQUFJLENBQUMsUUFBUSxHQUFHLE1BQU0sSUFBSSxDQUFDLHNCQUFzQixDQUMvQyxJQUFJLENBQUMsSUFBSSxFQUNULElBQUksQ0FBQyxHQUFHLEVBQ1IsUUFBUSxDQUFDLE1BQU0sQ0FDaEIsQ0FBQztJQUNKLENBQUM7SUFFRDs7Ozs7O09BTUc7SUFDSyxLQUFLLENBQUMsc0JBQXNCLENBQUMsSUFBUyxFQUFFLEdBQVcsRUFBRSxNQUFjO1FBQ3pFLE9BQU8sS0FBSyxDQUFDLEdBQUcsRUFBRTtZQUNoQixPQUFPLEVBQUU7Z0JBQ1AsbUVBQW1FO2dCQUNuRSxLQUFLLEVBQUUsU0FBUyxNQUFNLEdBQUc7YUFDMUI7WUFDRCxNQUFNLEVBQUUsSUFBSSxDQUFDLGVBQWUsQ0FBQyxNQUFNO1NBQ3BDLENBQUMsQ0FBQyxJQUFJLENBQUMsS0FBSyxFQUFDLFFBQVEsRUFBQyxFQUFFO1lBQ3ZCLElBQUksQ0FBQyxRQUFRLENBQUMsSUFBSSxFQUFFLENBQUM7Z0JBQ25CLE1BQU0sSUFBSSxLQUFLLENBQUMsdUJBQXVCLENBQUMsQ0FBQztZQUMzQyxDQUFDO1lBRUQsTUFBTSxNQUFNLEdBQUcsUUFBUSxDQUFDLElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQztZQUN6QyxNQUFNLElBQUksR0FBRyxJQUFJLFdBQVcsQ0FBQyxJQUFJLEVBQUUsR0FBRyxFQUFFLEdBQUUsQ0FBQyxFQUFFLE1BQU0sQ0FBQyxDQUFDO1lBRXJELE9BQU8sS0FBSyxJQUFJLEVBQUU7Z0JBQ2hCLE9BQU8sTUFBTSxDQUFDLElBQUksRUFBRSxDQUFDLElBQUksQ0FBQyxDQUFDLEVBQUMsSUFBSSxFQUFFLEtBQUssRUFBQyxFQUFFLEVBQUU7b0JBQzFDLG1CQUFtQjtvQkFDbkIsSUFBSSxJQUFJLEVBQUUsQ0FBQzt3QkFDVCxJQUFJLENBQUMsZ0JBQWdCLEdBQUcsSUFBSSxDQUFDO3dCQUM3QixJQUFJLENBQUMsZUFBZSxDQUFDLEtBQUssRUFBRSxDQUFDO3dCQUM3QixJQUFJLENBQUMsT0FBTyxDQUFDLEtBQUssRUFBRSxDQUFDO3dCQUNyQixJQUFJLENBQUMsS0FBSyxFQUFFLENBQUM7d0JBQ2IsT0FBTztvQkFDVCxDQUFDO29CQUVELElBQUksQ0FBQyxLQUFLLENBQUMsS0FBSyxDQUFDLENBQUM7Z0JBQ3BCLENBQUMsQ0FBQyxDQUFDO1lBQ0wsQ0FBQyxDQUFDO1FBQ0osQ0FBQyxDQUFDLENBQUM7SUFDTCxDQUFDO0lBRUQ7OztPQUdHO0lBQ0ssU0FBUyxDQUFDLFFBQWEsRUFBRSxRQUFhLEVBQUUsT0FBWTtRQUMxRCxLQUFLLE1BQU0sTUFBTSxJQUFJLE9BQU8sRUFBRSxDQUFDO1lBQzdCLE1BQU0sS0FBSyxHQUFHLElBQUksaUJBQWlCLENBQUM7Z0JBQ2xDLElBQUksRUFBRSxNQUFNLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLE9BQU87Z0JBQ3RDLFNBQVMsRUFBRSxDQUFDLEdBQUcsR0FBRyxNQUFNLENBQUMsR0FBRyxDQUFDLEdBQUcsTUFBTSxDQUFDLFNBQVM7Z0JBQ2hELFFBQVEsRUFBRSxDQUFDLEdBQUcsR0FBRyxNQUFNLENBQUMsUUFBUSxDQUFDLEdBQUcsTUFBTSxDQUFDLFNBQVM7Z0JBQ3BELElBQUksRUFBRSxNQUFNLENBQUMsSUFBSTthQUNsQixDQUFDLENBQUM7WUFDSCxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUM7WUFDakIsSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLENBQUM7WUFFM0IsTUFBTSxVQUFVLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLENBQUM7WUFDdEQsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxZQUFZLENBQUMsVUFBVSxDQUFDLEVBQUUsQ0FBQyxDQUFDO1lBQ25ELElBQUksQ0FBQyxJQUFJLENBQUMsYUFBYSxDQUFDLElBQUksRUFBRSxNQUFNLENBQUMsTUFBTSxDQUFDLENBQUM7UUFDL0MsQ0FBQztJQUNILENBQUM7SUFFRDs7O09BR0c7SUFDSyxPQUFPLENBQUMsS0FBaUI7UUFDL0IsSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDO1FBRWpCLHFEQUFxRDtRQUNyRCxNQUFNLGNBQWMsR0FDbEIsSUFBSSxDQUFDLElBQUksQ0FBQyxTQUFTLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLENBQUMsU0FBUyxDQUFDO1FBQ3JFLE1BQU0sYUFBYSxHQUFHLElBQUksQ0FBQyxTQUFTLEdBQUcsY0FBYyxDQUFDO1FBQ3RELE1BQU0sY0FBYyxHQUFHLEtBQUssQ0FBQyxTQUFTLEdBQUcsR0FBRyxDQUFDO1FBQzdDLElBQUksY0FBYyxHQUFHLGFBQWEsRUFBRSxDQUFDO1lBQ25DLEtBQUssQ0FBQyxLQUFLLEVBQUUsQ0FBQztZQUNkLE9BQU87UUFDVCxDQUFDO1FBRUQsNkNBQTZDO1FBQzdDLE1BQU0sb0JBQW9CLEdBQ3hCLElBQUksQ0FBQyxJQUFJLENBQUMsZUFBZTtZQUN6QixJQUFJLENBQUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxlQUFlLENBQUM7UUFDckQsTUFBTSxjQUFjLEdBQUcsY0FBYyxHQUFHLG9CQUFvQixDQUFDO1FBQzdELElBQUksY0FBYyxHQUFHLGNBQWMsRUFBRSxDQUFDO1lBQ3BDLEtBQUssQ0FBQyxLQUFLLEVBQUUsQ0FBQztZQUNkLElBQUksQ0FBQyxJQUFJLEdBQUcsSUFBSSxDQUFDO1lBQ2pCLElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUM7WUFDckIsT0FBTztRQUNULENBQUM7UUFFRCxJQUFJLENBQUMsV0FBVyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQztJQUMvQixDQUFDO0lBRU8sS0FBSyxDQUFDLGNBQWM7UUFDMUIsMENBQTBDO1FBQzFDLE9BQU8sSUFBSSxDQUFDLFdBQVcsQ0FBQyxNQUFNLEtBQUssQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLGdCQUFnQixFQUFFLENBQUM7WUFDL0QsTUFBTSxJQUFJLENBQUMsUUFBUSxFQUFFLENBQUM7WUFDdEIsTUFBTSxJQUFJLE9BQU8sQ0FBQyxHQUFHLENBQUMsRUFBRSxDQUFDLFVBQVUsQ0FBQyxHQUFHLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQztRQUMvQyxDQUFDO1FBRUQsNENBQTRDO1FBQzVDLElBQUksSUFBSSxDQUFDLFdBQVcsQ0FBQyxNQUFNLEtBQUssQ0FBQyxJQUFJLElBQUksQ0FBQyxTQUFTLEdBQUcsQ0FBQyxFQUFFLENBQUM7WUFDeEQsSUFBSSxhQUFhLEdBQUcsSUFBSSxDQUFDO1lBQ3pCLE9BQU8sSUFBSSxDQUFDLFdBQVcsQ0FBQyxNQUFNLEtBQUssQ0FBQyxFQUFFLENBQUM7Z0JBQ3JDLE1BQU0sSUFBSSxPQUFPLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxVQUFVLENBQUMsR0FBRyxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUM7Z0JBQzlDLGFBQWEsRUFBRSxDQUFDO2dCQUVoQixJQUFJLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztvQkFDZCxPQUFPO2dCQUNULENBQUM7Z0JBRUQsSUFBSSxhQUFhLEtBQUssQ0FBQyxFQUFFLENBQUM7b0JBQ3hCLE1BQU0sSUFBSSxLQUFLLENBQ2IsNEVBQTRFLElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FDN0YsQ0FBQztnQkFDSixDQUFDO1lBQ0gsQ0FBQztRQUNILENBQUM7SUFDSCxDQUFDO0lBRU0sS0FBSyxDQUFDLFlBQVk7UUFDdkIsTUFBTSxJQUFJLENBQUMsY0FBYyxFQUFFLENBQUM7UUFDNUIsTUFBTSxLQUFLLEdBQUcsSUFBSSxDQUFDLFdBQVcsQ0FBQyxLQUFLLEVBQUUsQ0FBQztRQUN2QyxJQUFJLEtBQUssRUFBRSxDQUFDO1lBQ1YsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO1FBQ3pCLENBQUM7UUFDRCxPQUFPLEtBQUssQ0FBQztJQUNmLENBQUM7SUFFRDs7T0FFRztJQUNJLEtBQUssQ0FBQyxLQUFLO1FBQ2hCLElBQUksQ0FBQyxlQUFlLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDN0IsSUFBSSxDQUFDLFdBQVcsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQyxLQUFLLENBQUMsS0FBSyxFQUFFLENBQUMsQ0FBQztRQUNqRCxJQUFJLENBQUM7WUFDSCxJQUFJLElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxLQUFLLFlBQVksRUFBRSxDQUFDO2dCQUN4QyxNQUFNLElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUM7Z0JBQzNCLElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUM7WUFDdkIsQ0FBQztRQUNILENBQUM7UUFBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQ1gsU0FBUztRQUNYLENBQUM7SUFDSCxDQUFDO0lBRU0sa0JBQWtCO1FBQ3ZCLE9BQU8sSUFBSSxDQUFDLGVBQWUsQ0FBQztJQUM5QixDQUFDO0lBRU0sWUFBWTtRQUNqQixPQUFPLElBQUksQ0FBQyxTQUFTLENBQUM7SUFDeEIsQ0FBQztDQUNGIn0=

package/lib/utils/video/parser/sink.d.ts

@@ -0,0 +1,9 @@
+export declare class MP4FileSink {
+    private setStatus;
+    private file;
+    private offset;
+    constructor(file: any, setStatus: any, offset?: number);
+    write(chunk: any): void;
+    close(): void;
+}
+//# sourceMappingURL=sink.d.ts.map

package/lib/utils/video/parser/sink.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sink.d.ts","sourceRoot":"","sources":["../../../../src/lib/utils/video/parser/sink.ts"],"names":[],"mappings":"AACA,qBAAa,WAAW;IACtB,OAAO,CAAC,SAAS,CAAC;IAClB,OAAO,CAAC,IAAI,CAAC;IACb,OAAO,CAAC,MAAM,CAAC;gBAEI,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,GAAE,MAAU;IAMzD,KAAK,CAAC,KAAK,EAAE,GAAG;IAYhB,KAAK;CAIb"}

package/lib/utils/video/parser/sink.js

@@ -0,0 +1,22 @@
+// Wraps an MP4Box File as a WritableStream underlying sink.
+export class MP4FileSink {
+    constructor(file, setStatus, offset = 0) {
+        this.file = file;
+        this.setStatus = setStatus;
+        this.offset = offset;
+    }
+    write(chunk) {
+        const buffer = new ArrayBuffer(chunk.byteLength);
+        new Uint8Array(buffer).set(chunk);
+        // Inform MP4Box where in the file this chunk is from.
+        buffer.fileStart = this.offset;
+        this.offset += buffer.byteLength;
+        this.setStatus('fetch', (this.offset / 1024 ** 2).toFixed(1) + ' MiB');
+        this.file.appendBuffer(buffer);
+    }
+    close() {
+        this.setStatus('fetch', 'Done');
+        this.file.flush();
+    }
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2luay5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uL3NyYy9saWIvdXRpbHMvdmlkZW8vcGFyc2VyL3NpbmsudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsNERBQTREO0FBQzVELE1BQU0sT0FBTyxXQUFXO0lBS3RCLFlBQW1CLElBQVMsRUFBRSxTQUFjLEVBQUUsU0FBaUIsQ0FBQztRQUM5RCxJQUFJLENBQUMsSUFBSSxHQUFHLElBQUksQ0FBQztRQUNqQixJQUFJLENBQUMsU0FBUyxHQUFHLFNBQVMsQ0FBQztRQUMzQixJQUFJLENBQUMsTUFBTSxHQUFHLE1BQU0sQ0FBQztJQUN2QixDQUFDO0lBRU0sS0FBSyxDQUFDLEtBQVU7UUFDckIsTUFBTSxNQUFNLEdBQUcsSUFBSSxXQUFXLENBQUMsS0FBSyxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ2pELElBQUksVUFBVSxDQUFDLE1BQU0sQ0FBQyxDQUFDLEdBQUcsQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUVsQyxzREFBc0Q7UUFDckQsTUFBYyxDQUFDLFNBQVMsR0FBRyxJQUFJLENBQUMsTUFBTSxDQUFDO1FBQ3hDLElBQUksQ0FBQyxNQUFNLElBQUksTUFBTSxDQUFDLFVBQVUsQ0FBQztRQUVqQyxJQUFJLENBQUMsU0FBUyxDQUFDLE9BQU8sRUFBRSxDQUFDLElBQUksQ0FBQyxNQUFNLEdBQUcsSUFBSSxJQUFJLENBQUMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsR0FBRyxNQUFNLENBQUMsQ0FBQztRQUN2RSxJQUFJLENBQUMsSUFBSSxDQUFDLFlBQVksQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUNqQyxDQUFDO0lBRU0sS0FBSztRQUNWLElBQUksQ0FBQyxTQUFTLENBQUMsT0FBTyxFQUFFLE1BQU0sQ0FBQyxDQUFDO1FBQ2hDLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDcEIsQ0FBQztDQUNGIn0=

package/lib/utils/video/parser/utils.d.ts

@@ -0,0 +1,10 @@
+export declare function description(file: any, track: any): Uint8Array;
+export interface Edit {
+    mediaTime: number;
+    segmentDuration: number;
+    mediaRateInteger: number;
+    mediaRateFraction: number;
+    fps: number;
+}
+export declare function mp4BoxEditToEdit(edit: any): Omit<Edit, 'fps'>;
+//# sourceMappingURL=utils.d.ts.map

package/lib/utils/video/parser/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../../src/lib/utils/video/parser/utils.ts"],"names":[],"mappings":"AAEA,wBAAgB,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,cAWhD;AAED,MAAM,WAAW,IAAI;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,GAAG,EAAE,MAAM,CAAC;CACb;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAO7D"}

package/lib/utils/video/parser/utils.js

@@ -0,0 +1,22 @@
+import { DataStream } from 'mp4box';
+export function description(file, track) {
+    const trak = file.getTrackById(track.id);
+    for (const entry of trak.mdia.minf.stbl.stsd.entries) {
+        const box = entry.avcC || entry.hvcC || entry.vpcC || entry.av1C;
+        if (box) {
+            const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
+            box.write(stream);
+            return new Uint8Array(stream.buffer, 8); // Remove the box header.
+        }
+    }
+    throw new Error('avcC, hvcC, vpcC, or av1C box not found');
+}
+export function mp4BoxEditToEdit(edit) {
+    return {
+        mediaTime: edit.media_time,
+        segmentDuration: edit.segment_duration,
+        mediaRateInteger: edit.media_rate_integer,
+        mediaRateFraction: edit.media_rate_fraction,
+    };
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidXRpbHMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi8uLi9zcmMvbGliL3V0aWxzL3ZpZGVvL3BhcnNlci91dGlscy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUMsVUFBVSxFQUFDLE1BQU0sUUFBUSxDQUFDO0FBRWxDLE1BQU0sVUFBVSxXQUFXLENBQUMsSUFBUyxFQUFFLEtBQVU7SUFDL0MsTUFBTSxJQUFJLEdBQUcsSUFBSSxDQUFDLFlBQVksQ0FBQyxLQUFLLENBQUMsRUFBRSxDQUFDLENBQUM7SUFDekMsS0FBSyxNQUFNLEtBQUssSUFBSSxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ3JELE1BQU0sR0FBRyxHQUFHLEtBQUssQ0FBQyxJQUFJLElBQUksS0FBSyxDQUFDLElBQUksSUFBSSxLQUFLLENBQUMsSUFBSSxJQUFJLEtBQUssQ0FBQyxJQUFJLENBQUM7UUFDakUsSUFBSSxHQUFHLEVBQUUsQ0FBQztZQUNSLE1BQU0sTUFBTSxHQUFHLElBQUksVUFBVSxDQUFDLFNBQVMsRUFBRSxDQUFDLEVBQUUsVUFBVSxDQUFDLFVBQVUsQ0FBQyxDQUFDO1lBQ25FLEdBQUcsQ0FBQyxLQUFLLENBQUMsTUFBTSxDQUFDLENBQUM7WUFDbEIsT0FBTyxJQUFJLFVBQVUsQ0FBQyxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUMseUJBQXlCO1FBQ3BFLENBQUM7SUFDSCxDQUFDO0lBQ0QsTUFBTSxJQUFJLEtBQUssQ0FBQyx5Q0FBeUMsQ0FBQyxDQUFDO0FBQzdELENBQUM7QUFVRCxNQUFNLFVBQVUsZ0JBQWdCLENBQUMsSUFBUztJQUN4QyxPQUFPO1FBQ0wsU0FBUyxFQUFFLElBQUksQ0FBQyxVQUFVO1FBQzFCLGVBQWUsRUFBRSxJQUFJLENBQUMsZ0JBQWdCO1FBQ3RDLGdCQUFnQixFQUFFLElBQUksQ0FBQyxrQkFBa0I7UUFDekMsaUJBQWlCLEVBQUUsSUFBSSxDQUFDLG1CQUFtQjtLQUM1QyxDQUFDO0FBQ0osQ0FBQyJ9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@revideo/2d",
-  "version": "0.4.7-two.1024+6b376d6f",
+  "version": "0.4.8-alpha.1032+c87f8411",
   "description": "A 2D renderer for revideo",
   "author": "revideo",
   "homepage": "https://re.video/",
@@ -30,7 +30,7 @@
   "devDependencies": {
     "@preact/signals": "^1.2.1",
     "@revideo/internal": "^0.0.0",
-    "@revideo/ui": "^0.4.7-two.1024+6b376d6f",
+    "@revideo/ui": "^0.4.8-alpha.1032+c87f8411",
     "clsx": "^2.0.0",
     "jsdom": "^22.1.0",
     "preact": "^10.19.2",
@@ -40,12 +40,12 @@
     "@codemirror/language": "^6.10.1",
     "@lezer/common": "^1.2.1",
     "@lezer/highlight": "^1.2.0",
-    "@revideo/core": "^0.4.7-two.1024+6b376d6f",
+    "@revideo/core": "^0.4.8-alpha.1032+c87f8411",
     "code-fns": "^0.8.2",
     "hls.js": "^1.5.11",
     "mathjax-full": "^3.2.2",
     "mp4box": "^0.5.2",
     "parse-svg-path": "^0.1.2"
   },
-  "gitHead": "6b376d6fd166f21e381296f17f0a8f89e2db83ca"
+  "gitHead": "c87f841121fcdf09215d782516482ee0090ca11a"
 }

package/src/lib/components/Video.ts

@@ -12,7 +12,7 @@ import {computed, initial, nodeName, signal} from '../decorators';
 import {DesiredLength} from '../partials';
 import {drawImage} from '../utils';
 import {ImageCommunication} from '../utils/video/ffmpeg-client';
-import {getFrame} from '../utils/video/mp4-parser-manager';
+import {dropExtractor, getFrame} from '../utils/video/mp4-parser-manager';
 import {Media, MediaProps} from './Media';
 
 export interface VideoProps extends MediaProps {
@@ -195,7 +195,8 @@ export class Video extends Media {
       }
     }
 
-    if (Math.abs(video.currentTime - time) > 0.3) {
+    // reseek when video is out of sync by more than one second
+    if (Math.abs(video.currentTime - time) > 1) {
       this.setCurrentTime(time);
     } else if (!playing) {
       video.currentTime = time;
@@ -326,6 +327,12 @@ export class Video extends Media {
     ).toString();
   }
 
+  public override remove() {
+    super.remove();
+    dropExtractor(this.key, this.video().src);
+    return this;
+  }
+
   private async detectFileType(): Promise<
     'mp4' | 'webm' | 'hls' | 'mov' | 'unknown'
   > {

package/src/lib/components/index.ts

@@ -11,6 +11,7 @@ export * from './Knot';
 export * from './Latex';
 export * from './Layout';
 export * from './Line';
+export * from './Media';
 export * from './Node';
 export * from './Path';
 export * from './Polygon';

package/src/lib/scenes/Scene2D.ts

@@ -11,7 +11,7 @@ import {
   Vector2,
   useLogger,
 } from '@revideo/core';
-import {Audio, Node, Video, View2D} from '../components';
+import {Audio, Media, Node, Video, View2D} from '../components';
 
 export class Scene2D extends GeneratorScene<View2D> implements Inspectable {
   private view: View2D | null = null;
@@ -199,6 +199,16 @@ export class Scene2D extends GeneratorScene<View2D> implements Inspectable {
     return returnObjects;
   }
 
+  public override stopAllMedia() {
+    const playingMedia = Array.from(this.registeredNodes.values())
+      .filter((node): node is Media => node instanceof Media)
+      .filter(video => (video as Media).isPlaying());
+
+    for (const media of playingMedia) {
+      media.dispose();
+    }
+  }
+
   protected recreateView() {
     this.execute(() => {
       const size = this.getSize();

package/src/lib/utils/video/mp4-parser-manager.ts

@@ -1,7 +1,16 @@
-import {FrameExtractor} from './mp4-parser';
+import {Mp4Parser} from './parser';
 
 // List of VideoFrameExtractors
-const videoFrameExtractors = new Map<string, FrameExtractor>();
+const videoFrameExtractors = new Map<string, Mp4Parser>();
+
+export async function dropExtractor(id: string, filePath: string) {
+  const extractorId = filePath + '-' + id;
+  const extractor = videoFrameExtractors.get(extractorId);
+  if (extractor) {
+    extractor.close();
+    videoFrameExtractors.delete(extractorId);
+  }
+}
 
 export async function getFrame(
   id: string,
@@ -15,12 +24,22 @@ export async function getFrame(
 
   const frameDuration = 1 / fps;
 
+  /**
+   * Sometimes, HTMLVideoElement.duration is not accurate, which can lead to the
+   * requested time being greater than the duration.
+   * To prevent this, we clamp the time to the duration reported by the extractor.
+   */
+  const duration = extractor?.getDuration();
+  if (duration && time > duration) {
+    time = duration;
+  }
+
   const isOldFrame =
     extractor && Math.abs(time - extractor.getLastTime()) < frameDuration / 2;
 
   // If time has not changed, return the last frame
-  if (isOldFrame) {
-    const lastFrame = extractor!.getLastFrame();
+  if (extractor && isOldFrame) {
+    const lastFrame = extractor.getLastFrame();
     if (!lastFrame) {
       throw new Error('No last frame');
     }
@@ -43,7 +62,7 @@ export async function getFrame(
   }
 
   if (!extractor) {
-    extractor = new FrameExtractor(filePath, fps, time);
+    extractor = new Mp4Parser(filePath, fps, time);
     await extractor.start();
     videoFrameExtractors.set(extractorId, extractor);
   }

package/src/lib/utils/video/parser/index.ts

@@ -0,0 +1 @@
+export * from './parser';