@revideo/2d 0.4.2-turtle.993 → 0.4.3-alpha.1001

package/lib/utils/video/mp4-parser-manager.js

@@ -0,0 +1,38 @@
+import { FrameExtractor } from './mp4-parser';
+// List of VideoFrameExtractors
+const videoFrameExtractors = new Map();
+export async function getFrame(id, filePath, time, fps) {
+    // Check if we already have a VideoFrameExtractor for this video
+    const extractorId = filePath + '-' + id;
+    let extractor = videoFrameExtractors.get(extractorId);
+    const frameDuration = 1 / fps;
+    const isOldFrame = extractor && Math.abs(time - extractor.getLastTime()) < frameDuration / 2;
+    // If time has not changed, return the last frame
+    if (isOldFrame) {
+        const lastFrame = extractor.getLastFrame();
+        if (!lastFrame) {
+            throw new Error('No last frame');
+        }
+        return lastFrame;
+    }
+    // If the video has skipped back we need to create a new extractor
+    if (extractor && time + frameDuration < extractor.getTime()) {
+        extractor.close();
+        videoFrameExtractors.delete(extractorId);
+        extractor = undefined;
+    }
+    // If the video has skipped forward we need to create a new extractor
+    if (extractor && time > extractor.getTime() + frameDuration) {
+        extractor.close();
+        videoFrameExtractors.delete(extractorId);
+        extractor = undefined;
+    }
+    if (!extractor) {
+        extractor = new FrameExtractor(filePath, fps, time);
+        await extractor.start();
+        videoFrameExtractors.set(extractorId, extractor);
+    }
+    // Go to the frame that is closest to the requested time
+    return extractor.getNextFrame();
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibXA0LXBhcnNlci1tYW5hZ2VyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vLi4vc3JjL2xpYi91dGlscy92aWRlby9tcDQtcGFyc2VyLW1hbmFnZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFDLGNBQWMsRUFBQyxNQUFNLGNBQWMsQ0FBQztBQUU1QywrQkFBK0I7QUFDL0IsTUFBTSxvQkFBb0IsR0FBRyxJQUFJLEdBQUcsRUFBMEIsQ0FBQztBQUUvRCxNQUFNLENBQUMsS0FBSyxVQUFVLFFBQVEsQ0FDNUIsRUFBVSxFQUNWLFFBQWdCLEVBQ2hCLElBQVksRUFDWixHQUFXO0lBRVgsZ0VBQWdFO0lBQ2hFLE1BQU0sV0FBVyxHQUFHLFFBQVEsR0FBRyxHQUFHLEdBQUcsRUFBRSxDQUFDO0lBQ3hDLElBQUksU0FBUyxHQUFHLG9CQUFvQixDQUFDLEdBQUcsQ0FBQyxXQUFXLENBQUMsQ0FBQztJQUV0RCxNQUFNLGFBQWEsR0FBRyxDQUFDLEdBQUcsR0FBRyxDQUFDO0lBRTlCLE1BQU0sVUFBVSxHQUNkLFNBQVMsSUFBSSxJQUFJLENBQUMsR0FBRyxDQUFDLElBQUksR0FBRyxTQUFTLENBQUMsV0FBVyxFQUFFLENBQUMsR0FBRyxhQUFhLEdBQUcsQ0FBQyxDQUFDO0lBRTVFLGlEQUFpRDtJQUNqRCxJQUFJLFVBQVUsRUFBRSxDQUFDO1FBQ2YsTUFBTSxTQUFTLEdBQUcsU0FBVSxDQUFDLFlBQVksRUFBRSxDQUFDO1FBQzVDLElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQztZQUNmLE1BQU0sSUFBSSxLQUFLLENBQUMsZUFBZSxDQUFDLENBQUM7UUFDbkMsQ0FBQztRQUVELE9BQU8sU0FBUyxDQUFDO0lBQ25CLENBQUM7SUFFRCxrRUFBa0U7SUFDbEUsSUFBSSxTQUFTLElBQUksSUFBSSxHQUFHLGFBQWEsR0FBRyxTQUFTLENBQUMsT0FBTyxFQUFFLEVBQUUsQ0FBQztRQUM1RCxTQUFTLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDbEIsb0JBQW9CLENBQUMsTUFBTSxDQUFDLFdBQVcsQ0FBQyxDQUFDO1FBQ3pDLFNBQVMsR0FBRyxTQUFTLENBQUM7SUFDeEIsQ0FBQztJQUVELHFFQUFxRTtJQUNyRSxJQUFJLFNBQVMsSUFBSSxJQUFJLEdBQUcsU0FBUyxDQUFDLE9BQU8sRUFBRSxHQUFHLGFBQWEsRUFBRSxDQUFDO1FBQzVELFNBQVMsQ0FBQyxLQUFLLEVBQUUsQ0FBQztRQUNsQixvQkFBb0IsQ0FBQyxNQUFNLENBQUMsV0FBVyxDQUFDLENBQUM7UUFDekMsU0FBUyxHQUFHLFNBQVMsQ0FBQztJQUN4QixDQUFDO0lBRUQsSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDO1FBQ2YsU0FBUyxHQUFHLElBQUksY0FBYyxDQUFDLFFBQVEsRUFBRSxHQUFHLEVBQUUsSUFBSSxDQUFDLENBQUM7UUFDcEQsTUFBTSxTQUFTLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDeEIsb0JBQW9CLENBQUMsR0FBRyxDQUFDLFdBQVcsRUFBRSxTQUFTLENBQUMsQ0FBQztJQUNuRCxDQUFDO0lBRUQsd0RBQXdEO0lBQ3hELE9BQU8sU0FBUyxDQUFDLFlBQVksRUFBRSxDQUFDO0FBQ2xDLENBQUMifQ==

package/lib/utils/video/mp4-parser.d.ts

@@ -0,0 +1,63 @@
+export declare class FrameExtractor {
+    private frameBuffer;
+    private closed;
+    private uri;
+    private targetFps;
+    private sourceFps;
+    private startTime;
+    private framesRequested;
+    private framesDue;
+    private lastFrame;
+    private decoder;
+    private file;
+    private readMoreFromResponse;
+    private responseFinished;
+    constructor(uri: string, targetFps: number, startTime: number);
+    /**
+     * Starts streaming the video at the given URI from the given offset.
+     * @param file - MP4Box file. Needs to be created and configured before calling this function.
+     * @param uri - URI of the video file.
+     * @param offset - Offset to start streaming from.
+     * @returns - A function to read more data from the response.
+     */
+    private startStreamingAtOffset;
+    /**
+     * Loads the file at the given URI until it finds the moov box.
+     * Once found, it calls `setConfig` with the video configuration.
+     * @param uri - The URI of the video file.
+     * @param setConfig - Callback to set the video configuration.
+     * @returns
+     */
+    private getFileInfo;
+    /**
+     * Starts the extraction process.
+     */
+    start(): Promise<void>;
+    /**
+     * Called when we are done with the extractor.
+     */
+    close(): void;
+    /**
+     * @returns - Time in seconds of the current frame (using the target FPS)
+     */
+    getTime(): number;
+    /**
+     * @returns - Time in seconds of the last frame (using the target FPS)
+     */
+    getLastTime(): number;
+    getLastFrame(): VideoFrame | null;
+    /**
+     * Called when samples are available on the MP4 file.
+     * Sends chunks to the decoder.
+     */
+    private onSamples;
+    /**
+     * Called when the decoder has a frame ready.
+     * Pushes the frame to the buffer so it can be consumed.
+     */
+    private onFrame;
+    private sum;
+    private populateBuffer;
+    getNextFrame(): Promise<VideoFrame>;
+}
+//# sourceMappingURL=mp4-parser.d.ts.map

package/lib/utils/video/mp4-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mp4-parser.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/video/mp4-parser.ts"],"names":[],"mappings":"AA6CA,qBAAa,cAAc;IACzB,OAAO,CAAC,WAAW,CAAoB;IACvC,OAAO,CAAC,MAAM,CAAS;IAGvB,OAAO,CAAC,GAAG,CAAS;IAEpB,OAAO,CAAC,SAAS,CAAS;IAE1B,OAAO,CAAC,SAAS,CAAS;IAE1B,OAAO,CAAC,SAAS,CAAS;IAE1B,OAAO,CAAC,eAAe,CAAK;IAE5B,OAAO,CAAC,SAAS,CAAK;IAEtB,OAAO,CAAC,SAAS,CAA2B;IAE5C,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,IAAI,CAAM;IAElB,OAAO,CAAC,oBAAoB,CAAuC;IACnE,OAAO,CAAC,gBAAgB,CAAS;gBAEd,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IAgBpE;;;;;;OAMG;YACW,sBAAsB;IA8BpC;;;;;;OAMG;YACW,WAAW;IA0DzB;;OAEG;IACU,KAAK;IAelB;;OAEG;IACI,KAAK;IAOZ;;OAEG;IACI,OAAO;IAId;;OAEG;IACI,WAAW;IAIX,YAAY;IAInB;;;OAGG;IACH,OAAO,CAAC,SAAS;IAajB;;;OAGG;IACH,OAAO,CAAC,OAAO;IAiBf,OAAO,CAAC,GAAG,CAAK;YAEF,cAAc;IAuBf,YAAY,IAAI,OAAO,CAAC,UAAU,CAAC;CAwCjD"}

package/lib/utils/video/mp4-parser.js

@@ -0,0 +1,262 @@
+import { createFile, DataStream } from 'mp4box';
+// Wraps an MP4Box File as a WritableStream underlying sink.
+class MP4FileSink {
+    constructor(file, setStatus, offset = 0) {
+        this.file = file;
+        this.setStatus = setStatus;
+        this.offset = offset;
+    }
+    write(chunk) {
+        const buffer = new ArrayBuffer(chunk.byteLength);
+        new Uint8Array(buffer).set(chunk);
+        // Inform MP4Box where in the file this chunk is from.
+        buffer.fileStart = this.offset;
+        this.offset += buffer.byteLength;
+        this.setStatus('fetch', (this.offset / 1024 ** 2).toFixed(1) + ' MiB');
+        this.file.appendBuffer(buffer);
+    }
+    close() {
+        this.setStatus('fetch', 'Done');
+        this.file.flush();
+    }
+}
+function description(file, track) {
+    const trak = file.getTrackById(track.id);
+    for (const entry of trak.mdia.minf.stbl.stsd.entries) {
+        const box = entry.avcC || entry.hvcC || entry.vpcC || entry.av1C;
+        if (box) {
+            const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
+            box.write(stream);
+            return new Uint8Array(stream.buffer, 8); // Remove the box header.
+        }
+    }
+    throw new Error('avcC, hvcC, vpcC, or av1C box not found');
+}
+export class FrameExtractor {
+    constructor(uri, targetFps, startTime) {
+        this.frameBuffer = [];
+        this.closed = false;
+        // Number of frames that have been requested and returned.
+        this.framesRequested = 0;
+        // Number of samples that have been passed to the decoder but have not been output yet.
+        this.framesDue = 0;
+        // Last frame that has been output.
+        this.lastFrame = null;
+        this.readMoreFromResponse = async () => { };
+        this.responseFinished = false;
+        this.sum = 0;
+        this.uri = uri;
+        this.startTime = startTime;
+        // Initialized after the file is loaded.
+        this.sourceFps = 0;
+        this.targetFps = targetFps;
+        this.decoder = new VideoDecoder({
+            output: this.onFrame.bind(this),
+            error(e) {
+                console.error(e);
+            },
+        });
+    }
+    /**
+     * Starts streaming the video at the given URI from the given offset.
+     * @param file - MP4Box file. Needs to be created and configured before calling this function.
+     * @param uri - URI of the video file.
+     * @param offset - Offset to start streaming from.
+     * @returns - A function to read more data from the response.
+     */
+    async startStreamingAtOffset(file, uri, offset) {
+        return fetch(uri, {
+            headers: {
+                /* eslint-disable-next-line @typescript-eslint/naming-convention */
+                Range: `bytes=${offset}-`,
+            },
+        }).then(async (response) => {
+            if (!response.body) {
+                throw new Error('Response body is null');
+            }
+            const reader = response.body.getReader();
+            const sink = new MP4FileSink(file, () => { }, offset);
+            return async () => {
+                return reader.read().then(({ done, value }) => {
+                    // Request is done.
+                    if (done) {
+                        this.responseFinished = true;
+                        this.decoder.flush();
+                        sink.close();
+                        return;
+                    }
+                    sink.write(value);
+                });
+            };
+        });
+    }
+    /**
+     * Loads the file at the given URI until it finds the moov box.
+     * Once found, it calls `setConfig` with the video configuration.
+     * @param uri - The URI of the video file.
+     * @param setConfig - Callback to set the video configuration.
+     * @returns
+     */
+    async getFileInfo(uri, setConfig) {
+        return new Promise((res, rej) => {
+            const file = createFile();
+            let found = false;
+            file.onReady = (info) => {
+                found = true;
+                const track = info.videoTracks[0];
+                const config = {
+                    // Browser doesn't support parsing full vp8 codec (eg: `vp08.00.41.08`),
+                    // they only support `vp8`.
+                    codec: track.codec.startsWith('vp08') ? 'vp8' : track.codec,
+                    codedHeight: track.video.height,
+                    codedWidth: track.video.width,
+                    description: description(file, track),
+                };
+                // Calculate FPS
+                if (track.nb_samples && track.timescale && info.duration) {
+                    this.sourceFps = track.nb_samples / (info.duration / info.timescale);
+                }
+                setConfig(config);
+                file.setExtractionOptions(track.id);
+                file.start();
+                res(file);
+            };
+            return fetch(uri).then(async (response) => {
+                if (!response.body) {
+                    throw new Error('Response body is null');
+                }
+                const reader = response.body.getReader();
+                const sink = new MP4FileSink(file, () => { });
+                while (!found) {
+                    await reader.read().then(({ done, value }) => {
+                        if (done) {
+                            file.flush();
+                            rej('Could not find moov');
+                            return;
+                        }
+                        sink.write(value);
+                    });
+                }
+            });
+        });
+    }
+    /**
+     * Starts the extraction process.
+     */
+    async start() {
+        this.file = await this.getFileInfo(this.uri, this.decoder.configure.bind(this.decoder));
+        this.file.onSamples = this.onSamples.bind(this);
+        const seekInfo = this.file.seek(this.startTime, true);
+        this.readMoreFromResponse = await this.startStreamingAtOffset(this.file, this.uri, seekInfo.offset);
+    }
+    /**
+     * Called when we are done with the extractor.
+     */
+    close() {
+        this.closed = true;
+        this.file.flush();
+        this.frameBuffer.forEach(frame => frame.close());
+        this.lastFrame?.close();
+    }
+    /**
+     * @returns - Time in seconds of the current frame (using the target FPS)
+     */
+    getTime() {
+        return this.startTime + this.framesRequested / this.targetFps;
+    }
+    /**
+     * @returns - Time in seconds of the last frame (using the target FPS)
+     */
+    getLastTime() {
+        return this.startTime + (this.framesRequested - 1) / this.targetFps;
+    }
+    getLastFrame() {
+        return this.lastFrame;
+    }
+    /**
+     * Called when samples are available on the MP4 file.
+     * Sends chunks to the decoder.
+     */
+    onSamples(_unused1, _unused2, samples) {
+        for (const sample of samples) {
+            const chunk = new EncodedVideoChunk({
+                type: sample.is_sync ? 'key' : 'delta',
+                timestamp: (1e6 * sample.cts) / sample.timescale,
+                duration: (1e6 * sample.duration) / sample.timescale,
+                data: sample.data,
+            });
+            this.framesDue++;
+            this.decoder.decode(chunk);
+        }
+    }
+    /**
+     * Called when the decoder has a frame ready.
+     * Pushes the frame to the buffer so it can be consumed.
+     */
+    onFrame(frame) {
+        this.framesDue--;
+        const timestampInSeconds = frame.timestamp / 1e6;
+        if (timestampInSeconds < this.startTime) {
+            frame.close();
+            return;
+        }
+        // If the extractor is already closed, close the frame.
+        if (this.closed) {
+            frame.close();
+            return;
+        }
+        this.frameBuffer.push(frame);
+    }
+    async populateBuffer() {
+        // Fetch more frames if we don't have any.
+        while (this.frameBuffer.length === 0 && !this.responseFinished) {
+            await this.readMoreFromResponse();
+            await new Promise(res => setTimeout(res, 0));
+        }
+        // Wait for decoder if there are frames due.
+        if (this.frameBuffer.length === 0 && this.framesDue) {
+            let maxIterations = 1000;
+            while (this.frameBuffer.length === 0) {
+                await new Promise(res => setTimeout(res, 10));
+                maxIterations--;
+                if (maxIterations === 0) {
+                    throw new Error('Timed out while waiting for VideoDecoder to produce a frame.');
+                }
+            }
+        }
+    }
+    async getNextFrame() {
+        const samplingRate = this.sourceFps / this.targetFps;
+        this.sum += samplingRate;
+        if (this.sum <= 1 && this.lastFrame) {
+            this.framesRequested++;
+            return this.lastFrame;
+        }
+        await this.populateBuffer();
+        // We're at the end of the video and there are no more frames to extract.
+        if (this.frameBuffer.length === 0) {
+            return this.lastFrame;
+        }
+        while (this.sum >= 2) {
+            // Burn frame
+            const frame = this.frameBuffer.shift();
+            frame.close();
+            this.sum -= 1;
+            await this.populateBuffer();
+            if (this.frameBuffer.length === 0) {
+                return this.lastFrame;
+            }
+        }
+        if (this.sum >= 1 || !this.lastFrame) {
+            this.sum -= 1;
+            const frame = this.frameBuffer.shift();
+            this.lastFrame?.close();
+            this.lastFrame = frame;
+            this.framesRequested++;
+            return frame;
+        }
+        // One of the three if statements above is always true.
+        throw new Error('Unreachable code');
+    }
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibXA0LXBhcnNlci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9saWIvdXRpbHMvdmlkZW8vbXA0LXBhcnNlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUMsVUFBVSxFQUFFLFVBQVUsRUFBQyxNQUFNLFFBQVEsQ0FBQztBQUU5Qyw0REFBNEQ7QUFDNUQsTUFBTSxXQUFXO0lBS2YsWUFBbUIsSUFBUyxFQUFFLFNBQWMsRUFBRSxTQUFpQixDQUFDO1FBQzlELElBQUksQ0FBQyxJQUFJLEdBQUcsSUFBSSxDQUFDO1FBQ2pCLElBQUksQ0FBQyxTQUFTLEdBQUcsU0FBUyxDQUFDO1FBQzNCLElBQUksQ0FBQyxNQUFNLEdBQUcsTUFBTSxDQUFDO0lBQ3ZCLENBQUM7SUFFTSxLQUFLLENBQUMsS0FBVTtRQUNyQixNQUFNLE1BQU0sR0FBRyxJQUFJLFdBQVcsQ0FBQyxLQUFLLENBQUMsVUFBVSxDQUFDLENBQUM7UUFDakQsSUFBSSxVQUFVLENBQUMsTUFBTSxDQUFDLENBQUMsR0FBRyxDQUFDLEtBQUssQ0FBQyxDQUFDO1FBRWxDLHNEQUFzRDtRQUNyRCxNQUFjLENBQUMsU0FBUyxHQUFHLElBQUksQ0FBQyxNQUFNLENBQUM7UUFDeEMsSUFBSSxDQUFDLE1BQU0sSUFBSSxNQUFNLENBQUMsVUFBVSxDQUFDO1FBRWpDLElBQUksQ0FBQyxTQUFTLENBQUMsT0FBTyxFQUFFLENBQUMsSUFBSSxDQUFDLE1BQU0sR0FBRyxJQUFJLElBQUksQ0FBQyxDQUFDLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxHQUFHLE1BQU0sQ0FBQyxDQUFDO1FBQ3ZFLElBQUksQ0FBQyxJQUFJLENBQUMsWUFBWSxDQUFDLE1BQU0sQ0FBQyxDQUFDO0lBQ2pDLENBQUM7SUFFTSxLQUFLO1FBQ1YsSUFBSSxDQUFDLFNBQVMsQ0FBQyxPQUFPLEVBQUUsTUFBTSxDQUFDLENBQUM7UUFDaEMsSUFBSSxDQUFDLElBQUksQ0FBQyxLQUFLLEVBQUUsQ0FBQztJQUNwQixDQUFDO0NBQ0Y7QUFFRCxTQUFTLFdBQVcsQ0FBQyxJQUFTLEVBQUUsS0FBVTtJQUN4QyxNQUFNLElBQUksR0FBRyxJQUFJLENBQUMsWUFBWSxDQUFDLEtBQUssQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUN6QyxLQUFLLE1BQU0sS0FBSyxJQUFJLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7UUFDckQsTUFBTSxHQUFHLEdBQUcsS0FBSyxDQUFDLElBQUksSUFBSSxLQUFLLENBQUMsSUFBSSxJQUFJLEtBQUssQ0FBQyxJQUFJLElBQUksS0FBSyxDQUFDLElBQUksQ0FBQztRQUNqRSxJQUFJLEdBQUcsRUFBRSxDQUFDO1lBQ1IsTUFBTSxNQUFNLEdBQUcsSUFBSSxVQUFVLENBQUMsU0FBUyxFQUFFLENBQUMsRUFBRSxVQUFVLENBQUMsVUFBVSxDQUFDLENBQUM7WUFDbkUsR0FBRyxDQUFDLEtBQUssQ0FBQyxNQUFNLENBQUMsQ0FBQztZQUNsQixPQUFPLElBQUksVUFBVSxDQUFDLE1BQU0sQ0FBQyxNQUFNLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyx5QkFBeUI7UUFDcEUsQ0FBQztJQUNILENBQUM7SUFDRCxNQUFNLElBQUksS0FBSyxDQUFDLHlDQUF5QyxDQUFDLENBQUM7QUFDN0QsQ0FBQztBQUVELE1BQU0sT0FBTyxjQUFjO0lBeUJ6QixZQUFtQixHQUFXLEVBQUUsU0FBaUIsRUFBRSxTQUFpQjtRQXhCNUQsZ0JBQVcsR0FBaUIsRUFBRSxDQUFDO1FBQy9CLFdBQU0sR0FBRyxLQUFLLENBQUM7UUFVdkIsMERBQTBEO1FBQ2xELG9CQUFlLEdBQUcsQ0FBQyxDQUFDO1FBQzVCLHVGQUF1RjtRQUMvRSxjQUFTLEdBQUcsQ0FBQyxDQUFDO1FBQ3RCLG1DQUFtQztRQUMzQixjQUFTLEdBQXNCLElBQUksQ0FBQztRQUtwQyx5QkFBb0IsR0FBd0IsS0FBSyxJQUFJLEVBQUUsR0FBRSxDQUFDLENBQUM7UUFDM0QscUJBQWdCLEdBQUcsS0FBSyxDQUFDO1FBNE16QixRQUFHLEdBQUcsQ0FBQyxDQUFDO1FBek1kLElBQUksQ0FBQyxHQUFHLEdBQUcsR0FBRyxDQUFDO1FBQ2YsSUFBSSxDQUFDLFNBQVMsR0FBRyxTQUFTLENBQUM7UUFFM0Isd0NBQXdDO1FBQ3hDLElBQUksQ0FBQyxTQUFTLEdBQUcsQ0FBQyxDQUFDO1FBQ25CLElBQUksQ0FBQyxTQUFTLEdBQUcsU0FBUyxDQUFDO1FBRTNCLElBQUksQ0FBQyxPQUFPLEdBQUcsSUFBSSxZQUFZLENBQUM7WUFDOUIsTUFBTSxFQUFFLElBQUksQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQztZQUMvQixLQUFLLENBQUMsQ0FBQztnQkFDTCxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDO1lBQ25CLENBQUM7U0FDRixDQUFDLENBQUM7SUFDTCxDQUFDO0lBRUQ7Ozs7OztPQU1HO0lBQ0ssS0FBSyxDQUFDLHNCQUFzQixDQUFDLElBQVMsRUFBRSxHQUFXLEVBQUUsTUFBYztRQUN6RSxPQUFPLEtBQUssQ0FBQyxHQUFHLEVBQUU7WUFDaEIsT0FBTyxFQUFFO2dCQUNQLG1FQUFtRTtnQkFDbkUsS0FBSyxFQUFFLFNBQVMsTUFBTSxHQUFHO2FBQzFCO1NBQ0YsQ0FBQyxDQUFDLElBQUksQ0FBQyxLQUFLLEVBQUMsUUFBUSxFQUFDLEVBQUU7WUFDdkIsSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLEVBQUUsQ0FBQztnQkFDbkIsTUFBTSxJQUFJLEtBQUssQ0FBQyx1QkFBdUIsQ0FBQyxDQUFDO1lBQzNDLENBQUM7WUFFRCxNQUFNLE1BQU0sR0FBRyxRQUFRLENBQUMsSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDO1lBQ3pDLE1BQU0sSUFBSSxHQUFHLElBQUksV0FBVyxDQUFDLElBQUksRUFBRSxHQUFHLEVBQUUsR0FBRSxDQUFDLEVBQUUsTUFBTSxDQUFDLENBQUM7WUFFckQsT0FBTyxLQUFLLElBQUksRUFBRTtnQkFDaEIsT0FBTyxNQUFNLENBQUMsSUFBSSxFQUFFLENBQUMsSUFBSSxDQUFDLENBQUMsRUFBQyxJQUFJLEVBQUUsS0FBSyxFQUFDLEVBQUUsRUFBRTtvQkFDMUMsbUJBQW1CO29CQUNuQixJQUFJLElBQUksRUFBRSxDQUFDO3dCQUNULElBQUksQ0FBQyxnQkFBZ0IsR0FBRyxJQUFJLENBQUM7d0JBQzdCLElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxFQUFFLENBQUM7d0JBQ3JCLElBQUksQ0FBQyxLQUFLLEVBQUUsQ0FBQzt3QkFDYixPQUFPO29CQUNULENBQUM7b0JBRUQsSUFBSSxDQUFDLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQztnQkFDcEIsQ0FBQyxDQUFDLENBQUM7WUFDTCxDQUFDLENBQUM7UUFDSixDQUFDLENBQUMsQ0FBQztJQUNMLENBQUM7SUFFRDs7Ozs7O09BTUc7SUFDSyxLQUFLLENBQUMsV0FBVyxDQUN2QixHQUFXLEVBQ1gsU0FBK0M7UUFFL0MsT0FBTyxJQUFJLE9BQU8sQ0FBTSxDQUFDLEdBQUcsRUFBRSxHQUFHLEVBQUUsRUFBRTtZQUNuQyxNQUFNLElBQUksR0FBRyxVQUFVLEVBQUUsQ0FBQztZQUMxQixJQUFJLEtBQUssR0FBRyxLQUFLLENBQUM7WUFFbEIsSUFBSSxDQUFDLE9BQU8sR0FBRyxDQUFDLElBQVMsRUFBRSxFQUFFO2dCQUMzQixLQUFLLEdBQUcsSUFBSSxDQUFDO2dCQUViLE1BQU0sS0FBSyxHQUFHLElBQUksQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLENBQUM7Z0JBRWxDLE1BQU0sTUFBTSxHQUFHO29CQUNiLHdFQUF3RTtvQkFDeEUsMkJBQTJCO29CQUMzQixLQUFLLEVBQUUsS0FBSyxDQUFDLEtBQUssQ0FBQyxVQUFVLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLEtBQUs7b0JBQzNELFdBQVcsRUFBRSxLQUFLLENBQUMsS0FBSyxDQUFDLE1BQU07b0JBQy9CLFVBQVUsRUFBRSxLQUFLLENBQUMsS0FBSyxDQUFDLEtBQUs7b0JBQzdCLFdBQVcsRUFBRSxXQUFXLENBQUMsSUFBSSxFQUFFLEtBQUssQ0FBQztpQkFDdEMsQ0FBQztnQkFFRixnQkFBZ0I7Z0JBQ2hCLElBQUksS0FBSyxDQUFDLFVBQVUsSUFBSSxLQUFLLENBQUMsU0FBUyxJQUFJLElBQUksQ0FBQyxRQUFRLEVBQUUsQ0FBQztvQkFDekQsSUFBSSxDQUFDLFNBQVMsR0FBRyxLQUFLLENBQUMsVUFBVSxHQUFHLENBQUMsSUFBSSxDQUFDLFFBQVEsR0FBRyxJQUFJLENBQUMsU0FBUyxDQUFDLENBQUM7Z0JBQ3ZFLENBQUM7Z0JBRUQsU0FBUyxDQUFDLE1BQU0sQ0FBQyxDQUFDO2dCQUVsQixJQUFJLENBQUMsb0JBQW9CLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQyxDQUFDO2dCQUNwQyxJQUFJLENBQUMsS0FBSyxFQUFFLENBQUM7Z0JBRWIsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDO1lBQ1osQ0FBQyxDQUFDO1lBRUYsT0FBTyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsSUFBSSxDQUFDLEtBQUssRUFBQyxRQUFRLEVBQUMsRUFBRTtnQkFDdEMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLEVBQUUsQ0FBQztvQkFDbkIsTUFBTSxJQUFJLEtBQUssQ0FBQyx1QkFBdUIsQ0FBQyxDQUFDO2dCQUMzQyxDQUFDO2dCQUVELE1BQU0sTUFBTSxHQUFHLFFBQVEsQ0FBQyxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUM7Z0JBQ3pDLE1BQU0sSUFBSSxHQUFHLElBQUksV0FBVyxDQUFDLElBQUksRUFBRSxHQUFHLEVBQUUsR0FBRSxDQUFDLENBQUMsQ0FBQztnQkFFN0MsT0FBTyxDQUFDLEtBQUssRUFBRSxDQUFDO29CQUNkLE1BQU0sTUFBTSxDQUFDLElBQUksRUFBRSxDQUFDLElBQUksQ0FBQyxDQUFDLEVBQUMsSUFBSSxFQUFFLEtBQUssRUFBQyxFQUFFLEVBQUU7d0JBQ3pDLElBQUksSUFBSSxFQUFFLENBQUM7NEJBQ1QsSUFBSSxDQUFDLEtBQUssRUFBRSxDQUFDOzRCQUNiLEdBQUcsQ0FBQyxxQkFBcUIsQ0FBQyxDQUFDOzRCQUMzQixPQUFPO3dCQUNULENBQUM7d0JBRUQsSUFBSSxDQUFDLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQztvQkFDcEIsQ0FBQyxDQUFDLENBQUM7Z0JBQ0wsQ0FBQztZQUNILENBQUMsQ0FBQyxDQUFDO1FBQ0wsQ0FBQyxDQUFDLENBQUM7SUFDTCxDQUFDO0lBRUQ7O09BRUc7SUFDSSxLQUFLLENBQUMsS0FBSztRQUNoQixJQUFJLENBQUMsSUFBSSxHQUFHLE1BQU0sSUFBSSxDQUFDLFdBQVcsQ0FDaEMsSUFBSSxDQUFDLEdBQUcsRUFDUixJQUFJLENBQUMsT0FBTyxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUMxQyxDQUFDO1FBQ0YsSUFBSSxDQUFDLElBQUksQ0FBQyxTQUFTLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7UUFFaEQsTUFBTSxRQUFRLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsRUFBRSxJQUFJLENBQUMsQ0FBQztRQUN0RCxJQUFJLENBQUMsb0JBQW9CLEdBQUcsTUFBTSxJQUFJLENBQUMsc0JBQXNCLENBQzNELElBQUksQ0FBQyxJQUFJLEVBQ1QsSUFBSSxDQUFDLEdBQUcsRUFDUixRQUFRLENBQUMsTUFBTSxDQUNoQixDQUFDO0lBQ0osQ0FBQztJQUVEOztPQUVHO0lBQ0ksS0FBSztRQUNWLElBQUksQ0FBQyxNQUFNLEdBQUcsSUFBSSxDQUFDO1FBQ25CLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDbEIsSUFBSSxDQUFDLFdBQVcsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQyxLQUFLLENBQUMsS0FBSyxFQUFFLENBQUMsQ0FBQztRQUNqRCxJQUFJLENBQUMsU0FBUyxFQUFFLEtBQUssRUFBRSxDQUFDO0lBQzFCLENBQUM7SUFFRDs7T0FFRztJQUNJLE9BQU87UUFDWixPQUFPLElBQUksQ0FBQyxTQUFTLEdBQUcsSUFBSSxDQUFDLGVBQWUsR0FBRyxJQUFJLENBQUMsU0FBUyxDQUFDO0lBQ2hFLENBQUM7SUFFRDs7T0FFRztJQUNJLFdBQVc7UUFDaEIsT0FBTyxJQUFJLENBQUMsU0FBUyxHQUFHLENBQUMsSUFBSSxDQUFDLGVBQWUsR0FBRyxDQUFDLENBQUMsR0FBRyxJQUFJLENBQUMsU0FBUyxDQUFDO0lBQ3RFLENBQUM7SUFFTSxZQUFZO1FBQ2pCLE9BQU8sSUFBSSxDQUFDLFNBQVMsQ0FBQztJQUN4QixDQUFDO0lBRUQ7OztPQUdHO0lBQ0ssU0FBUyxDQUFDLFFBQWEsRUFBRSxRQUFhLEVBQUUsT0FBWTtRQUMxRCxLQUFLLE1BQU0sTUFBTSxJQUFJLE9BQU8sRUFBRSxDQUFDO1lBQzdCLE1BQU0sS0FBSyxHQUFHLElBQUksaUJBQWlCLENBQUM7Z0JBQ2xDLElBQUksRUFBRSxNQUFNLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLE9BQU87Z0JBQ3RDLFNBQVMsRUFBRSxDQUFDLEdBQUcsR0FBRyxNQUFNLENBQUMsR0FBRyxDQUFDLEdBQUcsTUFBTSxDQUFDLFNBQVM7Z0JBQ2hELFFBQVEsRUFBRSxDQUFDLEdBQUcsR0FBRyxNQUFNLENBQUMsUUFBUSxDQUFDLEdBQUcsTUFBTSxDQUFDLFNBQVM7Z0JBQ3BELElBQUksRUFBRSxNQUFNLENBQUMsSUFBSTthQUNsQixDQUFDLENBQUM7WUFDSCxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUM7WUFDakIsSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDN0IsQ0FBQztJQUNILENBQUM7SUFFRDs7O09BR0c7SUFDSyxPQUFPLENBQUMsS0FBaUI7UUFDL0IsSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDO1FBQ2pCLE1BQU0sa0JBQWtCLEdBQUcsS0FBSyxDQUFDLFNBQVMsR0FBRyxHQUFHLENBQUM7UUFDakQsSUFBSSxrQkFBa0IsR0FBRyxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUM7WUFDeEMsS0FBSyxDQUFDLEtBQUssRUFBRSxDQUFDO1lBQ2QsT0FBTztRQUNULENBQUM7UUFFRCx1REFBdUQ7UUFDdkQsSUFBSSxJQUFJLENBQUMsTUFBTSxFQUFFLENBQUM7WUFDaEIsS0FBSyxDQUFDLEtBQUssRUFBRSxDQUFDO1lBQ2QsT0FBTztRQUNULENBQUM7UUFFRCxJQUFJLENBQUMsV0FBVyxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQztJQUMvQixDQUFDO0lBSU8sS0FBSyxDQUFDLGNBQWM7UUFDMUIsMENBQTBDO1FBQzFDLE9BQU8sSUFBSSxDQUFDLFdBQVcsQ0FBQyxNQUFNLEtBQUssQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLGdCQUFnQixFQUFFLENBQUM7WUFDL0QsTUFBTSxJQUFJLENBQUMsb0JBQW9CLEVBQUUsQ0FBQztZQUNsQyxNQUFNLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsVUFBVSxDQUFDLEdBQUcsRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFDO1FBQy9DLENBQUM7UUFFRCw0Q0FBNEM7UUFDNUMsSUFBSSxJQUFJLENBQUMsV0FBVyxDQUFDLE1BQU0sS0FBSyxDQUFDLElBQUksSUFBSSxDQUFDLFNBQVMsRUFBRSxDQUFDO1lBQ3BELElBQUksYUFBYSxHQUFHLElBQUksQ0FBQztZQUN6QixPQUFPLElBQUksQ0FBQyxXQUFXLENBQUMsTUFBTSxLQUFLLENBQUMsRUFBRSxDQUFDO2dCQUNyQyxNQUFNLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUMsVUFBVSxDQUFDLEdBQUcsRUFBRSxFQUFFLENBQUMsQ0FBQyxDQUFDO2dCQUM5QyxhQUFhLEVBQUUsQ0FBQztnQkFFaEIsSUFBSSxhQUFhLEtBQUssQ0FBQyxFQUFFLENBQUM7b0JBQ3hCLE1BQU0sSUFBSSxLQUFLLENBQ2IsOERBQThELENBQy9ELENBQUM7Z0JBQ0osQ0FBQztZQUNILENBQUM7UUFDSCxDQUFDO0lBQ0gsQ0FBQztJQUVNLEtBQUssQ0FBQyxZQUFZO1FBQ3ZCLE1BQU0sWUFBWSxHQUFHLElBQUksQ0FBQyxTQUFTLEdBQUcsSUFBSSxDQUFDLFNBQVMsQ0FBQztRQUNyRCxJQUFJLENBQUMsR0FBRyxJQUFJLFlBQVksQ0FBQztRQUV6QixJQUFJLElBQUksQ0FBQyxHQUFHLElBQUksQ0FBQyxJQUFJLElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQztZQUNwQyxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7WUFDdkIsT0FBTyxJQUFJLENBQUMsU0FBUyxDQUFDO1FBQ3hCLENBQUM7UUFFRCxNQUFNLElBQUksQ0FBQyxjQUFjLEVBQUUsQ0FBQztRQUU1Qix5RUFBeUU7UUFDekUsSUFBSSxJQUFJLENBQUMsV0FBVyxDQUFDLE1BQU0sS0FBSyxDQUFDLEVBQUUsQ0FBQztZQUNsQyxPQUFPLElBQUksQ0FBQyxTQUFVLENBQUM7UUFDekIsQ0FBQztRQUVELE9BQU8sSUFBSSxDQUFDLEdBQUcsSUFBSSxDQUFDLEVBQUUsQ0FBQztZQUNyQixhQUFhO1lBQ2IsTUFBTSxLQUFLLEdBQUcsSUFBSSxDQUFDLFdBQVcsQ0FBQyxLQUFLLEVBQUcsQ0FBQztZQUN4QyxLQUFLLENBQUMsS0FBSyxFQUFFLENBQUM7WUFDZCxJQUFJLENBQUMsR0FBRyxJQUFJLENBQUMsQ0FBQztZQUVkLE1BQU0sSUFBSSxDQUFDLGNBQWMsRUFBRSxDQUFDO1lBQzVCLElBQUksSUFBSSxDQUFDLFdBQVcsQ0FBQyxNQUFNLEtBQUssQ0FBQyxFQUFFLENBQUM7Z0JBQ2xDLE9BQU8sSUFBSSxDQUFDLFNBQVUsQ0FBQztZQUN6QixDQUFDO1FBQ0gsQ0FBQztRQUVELElBQUksSUFBSSxDQUFDLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUM7WUFDckMsSUFBSSxDQUFDLEdBQUcsSUFBSSxDQUFDLENBQUM7WUFDZCxNQUFNLEtBQUssR0FBRyxJQUFJLENBQUMsV0FBVyxDQUFDLEtBQUssRUFBRyxDQUFDO1lBQ3hDLElBQUksQ0FBQyxTQUFTLEVBQUUsS0FBSyxFQUFFLENBQUM7WUFDeEIsSUFBSSxDQUFDLFNBQVMsR0FBRyxLQUFLLENBQUM7WUFDdkIsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO1lBQ3ZCLE9BQU8sS0FBSyxDQUFDO1FBQ2YsQ0FBQztRQUVELHVEQUF1RDtRQUN2RCxNQUFNLElBQUksS0FBSyxDQUFDLGtCQUFrQixDQUFDLENBQUM7SUFDdEMsQ0FBQztDQUNGIn0=

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@revideo/2d",
-  "version": "0.4.2-turtle.993+b11ad243",
+  "version": "0.4.3-alpha.1001+7bb63d94",
   "description": "A 2D renderer for revideo",
   "author": "revideo",
   "homepage": "https://re.video/",
@@ -30,7 +30,7 @@
   "devDependencies": {
     "@preact/signals": "^1.2.1",
     "@revideo/internal": "^0.0.0",
-    "@revideo/ui": "^0.4.2-turtle.993+b11ad243",
+    "@revideo/ui": "^0.4.3-alpha.1001+7bb63d94",
     "clsx": "^2.0.0",
     "jsdom": "^22.1.0",
     "preact": "^10.19.2",
@@ -40,10 +40,11 @@
     "@codemirror/language": "^6.10.1",
     "@lezer/common": "^1.2.1",
     "@lezer/highlight": "^1.2.0",
-    "@revideo/core": "^0.4.2-turtle.993+b11ad243",
+    "@revideo/core": "^0.4.3-alpha.1001+7bb63d94",
     "code-fns": "^0.8.2",
     "mathjax-full": "^3.2.2",
+    "mp4box": "^0.5.2",
     "parse-svg-path": "^0.1.2"
   },
-  "gitHead": "b11ad243419e053afcdc909981b917adf8f7d127"
+  "gitHead": "7bb63d94086f91d734e0a895489fad1d5c4b2478"
 }

package/src/lib/components/Video.ts

@@ -10,6 +10,8 @@ import {
 import {computed, initial, nodeName, signal} from '../decorators';
 import {DesiredLength} from '../partials';
 import {drawImage} from '../utils';
+import {ImageCommunication} from '../utils/video/ffmpeg-client';
+import {getFrame} from '../utils/video/mp4-parser-manager';
 import {Media, MediaProps} from './Media';
 
 export interface VideoProps extends MediaProps {
@@ -27,74 +29,6 @@ export interface VideoProps extends MediaProps {
   png?: SignalValue<boolean>;
 }
 
-class ImageCommunication {
-  public constructor() {
-    if (!import.meta.hot) {
-      throw new Error('FfmpegVideoFrame can only be used with HMR.');
-    }
-
-    import.meta.hot.on(
-      'revideo:ffmpeg-video-frame-res',
-      this.handler.bind(this),
-    );
-  }
-
-  private nextFrameHandlers: ((event: MessageEvent) => void)[] = [];
-
-  private handler(event: MessageEvent) {
-    const handlers = this.nextFrameHandlers;
-    this.nextFrameHandlers = [];
-
-    for (const handler of handlers) {
-      handler(event);
-    }
-  }
-
-  public async getFrame(
-    id: string,
-    src: string,
-    time: number,
-    duration: number,
-    fps: number,
-    png: boolean,
-  ) {
-    return new Promise<HTMLImageElement>((resolve, reject) => {
-      if (!import.meta.hot) {
-        reject('FfmpegVideoFrame can only be used with HMR.');
-        return;
-      }
-
-      function handler(event: MessageEvent) {
-        const image = new Image();
-
-        const uint8Array = new Uint8Array(event.data.frame.data);
-        const type = png ? 'image/png' : 'image/jpeg';
-        const blob = new Blob([uint8Array], {type});
-        const url = URL.createObjectURL(blob);
-
-        image.src = url;
-
-        image.onload = () => {
-          resolve(image);
-        };
-      }
-
-      this.nextFrameHandlers.push(handler);
-
-      import.meta.hot.send('revideo:ffmpeg-video-frame', {
-        data: {
-          id: id,
-          filePath: src,
-          startTime: time,
-          duration,
-          fps,
-          png,
-        },
-      });
-    });
-  }
-}
-
 @nodeName('Video')
 export class Video extends Media {
   /**
@@ -122,18 +56,19 @@ export class Video extends Media {
   public declare readonly smoothing: SimpleSignal<boolean, this>;
 
   /**
-   * Whether the video frames should be extracted as PNGs. Uses JPEGs when
-   * set to false.
-   *
-   * @remarks
-   * PNGs have better image quality and support transparency, but they make
-   * rendering slower.
+   * Which decoder to use during rendering. The `web` decoder is the fastest
+   * but only supports MP4 files. The `ffmpeg` decoder is slower and more resource
+   * intensive but supports more formats. The `slow` decoder is the slowest but
+   * supports all formats.
    *
-   * @defaultValue false
+   * @defaultValue null
    */
-  @initial(true)
+  @initial(null)
   @signal()
-  public declare readonly png: SimpleSignal<boolean, this>;
+  public declare readonly decoder: SimpleSignal<
+    'web' | 'ffmpeg' | 'slow' | null,
+    this
+  >;
 
   private static readonly pool: Record<string, HTMLVideoElement> = {};
 
@@ -143,6 +78,13 @@ export class Video extends Media {
 
   public constructor(props: VideoProps) {
     super(props);
+
+    // If the file is not mp4, warn.
+    if (!this.fullSource().endsWith('.mp4')) {
+      console.warn(
+        `WARNING: Video source is not an MP4 file. This may significantly slow down rendering: ${this.fullSource()}`,
+      );
+    }
   }
 
   protected override desiredSize(): SerializedVector2<DesiredLength> {
@@ -258,7 +200,21 @@ export class Video extends Media {
 
   protected lastFrame: HTMLImageElement | null = null;
 
-  protected async serverSeekedVideo(): Promise<HTMLImageElement> {
+  protected async webcodecSeekedVideo(): Promise<CanvasImageSource> {
+    const video = this.video();
+    const time = this.clampTime(this.time());
+
+    video.playbackRate = this.playbackRate();
+
+    if (this.lastFrame && this.lastTime === time) {
+      return this.lastFrame;
+    }
+
+    const fps = this.view().fps() / this.playbackRate();
+    return getFrame(this.key, video.src, time, fps);
+  }
+
+  protected async ffmpegSeekedVideo(): Promise<HTMLImageElement> {
     const video = this.video();
     const time = this.clampTime(this.time());
     const duration = this.getDuration();
@@ -281,7 +237,6 @@ export class Video extends Media {
       time,
       duration,
       fps,
-      this.png(),
     );
     this.lastFrame = frame;
     this.lastTime = time;
@@ -291,6 +246,8 @@ export class Video extends Media {
 
   protected async seekFunction() {
     const playbackState = this.view().playbackState();
+
+    // During playback
     if (
       playbackState === PlaybackState.Playing ||
       playbackState === PlaybackState.Presenting
@@ -298,8 +255,30 @@ export class Video extends Media {
       return this.fastSeekedVideo();
     }
 
-    if (playbackState === PlaybackState.Rendering) {
-      return this.serverSeekedVideo();
+    if (playbackState === PlaybackState.Paused) {
+      return this.seekedVideo();
+    }
+
+    // During rendering, if set explicitly
+    if (this.decoder() === 'slow') {
+      return this.seekedVideo();
+    }
+
+    if (this.decoder() === 'ffmpeg') {
+      return this.ffmpegSeekedVideo();
+    }
+
+    if (this.decoder() === 'web') {
+      return this.webcodecSeekedVideo();
+    }
+
+    // If not set explicitly, use file extension to determine decoder
+    if (this.src().endsWith('.mp4')) {
+      return this.webcodecSeekedVideo();
+    }
+
+    if (this.src().endsWith('.webm')) {
+      return this.ffmpegSeekedVideo();
     }
 
     return this.seekedVideo();

package/src/lib/utils/video/declarations.d.ts

@@ -0,0 +1 @@
+declare module 'mp4box';

package/src/lib/utils/video/ffmpeg-client.ts

@@ -0,0 +1,64 @@
+export class ImageCommunication {
+  public constructor() {
+    if (!import.meta.hot) {
+      throw new Error('FfmpegVideoFrame can only be used with HMR.');
+    }
+
+    import.meta.hot.on(
+      'revideo:ffmpeg-video-frame-res',
+      this.handler.bind(this),
+    );
+  }
+
+  private nextFrameHandlers: ((event: MessageEvent) => void)[] = [];
+
+  private handler(event: MessageEvent) {
+    const handlers = this.nextFrameHandlers;
+    this.nextFrameHandlers = [];
+
+    for (const handler of handlers) {
+      handler(event);
+    }
+  }
+
+  public async getFrame(
+    id: string,
+    src: string,
+    time: number,
+    duration: number,
+    fps: number,
+  ) {
+    return new Promise<HTMLImageElement>((resolve, reject) => {
+      if (!import.meta.hot) {
+        reject('FfmpegVideoFrame can only be used with HMR.');
+        return;
+      }
+
+      function handler(event: MessageEvent) {
+        const image = new Image();
+
+        const uint8Array = new Uint8Array(event.data.frame.data);
+        const blob = new Blob([uint8Array], {type: 'image/png'});
+        const url = URL.createObjectURL(blob);
+
+        image.src = url;
+
+        image.onload = () => {
+          resolve(image);
+        };
+      }
+
+      this.nextFrameHandlers.push(handler);
+
+      import.meta.hot.send('revideo:ffmpeg-video-frame', {
+        data: {
+          id: id,
+          filePath: src,
+          startTime: time,
+          duration,
+          fps,
+        },
+      });
+    });
+  }
+}