@remotion/media 4.0.351

package/dist/video-extraction/extract-frame-via-broadcast-channel.js

@@ -0,0 +1,107 @@
+import { extractFrameAndAudio } from '../extract-frame-and-audio';
+// Doesn't exist in studio
+if (window.remotion_broadcastChannel && window.remotion_isMainTab) {
+    window.remotion_broadcastChannel.addEventListener('message', async (event) => {
+        const data = event.data;
+        if (data.type === 'request') {
+            try {
+                const { frame, audio } = await extractFrameAndAudio({
+                    src: data.src,
+                    timeInSeconds: data.timeInSeconds,
+                    logLevel: data.logLevel,
+                    durationInSeconds: data.durationInSeconds,
+                    includeAudio: data.includeAudio,
+                    includeVideo: data.includeVideo,
+                    volume: data.volume,
+                    loop: data.loop,
+                });
+                const videoFrame = frame;
+                const imageBitmap = videoFrame
+                    ? await createImageBitmap(videoFrame)
+                    : null;
+                if (videoFrame) {
+                    videoFrame.close();
+                }
+                const response = {
+                    type: 'response-success',
+                    id: data.id,
+                    frame: imageBitmap,
+                    audio,
+                };
+                window.remotion_broadcastChannel.postMessage(response);
+                videoFrame?.close();
+            }
+            catch (error) {
+                const response = {
+                    type: 'response-error',
+                    id: data.id,
+                    errorStack: error.stack ?? 'No stack trace',
+                };
+                window.remotion_broadcastChannel.postMessage(response);
+            }
+        }
+        else {
+            throw new Error('Invalid message: ' + JSON.stringify(data));
+        }
+    });
+}
+export const extractFrameViaBroadcastChannel = ({ src, timeInSeconds, logLevel, durationInSeconds, includeAudio, includeVideo, isClientSideRendering, volume, loop, }) => {
+    if (isClientSideRendering || window.remotion_isMainTab) {
+        return extractFrameAndAudio({
+            logLevel,
+            src,
+            timeInSeconds,
+            durationInSeconds,
+            includeAudio,
+            includeVideo,
+            volume,
+            loop,
+        });
+    }
+    const requestId = crypto.randomUUID();
+    const resolvePromise = new Promise((resolve, reject) => {
+        const onMessage = (event) => {
+            const data = event.data;
+            if (!data) {
+                return;
+            }
+            if (data.type === 'response-success' && data.id === requestId) {
+                resolve({
+                    frame: data.frame ? data.frame : null,
+                    audio: data.audio ? data.audio : null,
+                });
+                window.remotion_broadcastChannel.removeEventListener('message', onMessage);
+            }
+            else if (data.type === 'response-error' && data.id === requestId) {
+                reject(data.errorStack);
+                window.remotion_broadcastChannel.removeEventListener('message', onMessage);
+            }
+        };
+        window.remotion_broadcastChannel.addEventListener('message', onMessage);
+    });
+    const request = {
+        type: 'request',
+        src,
+        timeInSeconds,
+        id: requestId,
+        logLevel,
+        durationInSeconds,
+        includeAudio,
+        includeVideo,
+        volume,
+        loop,
+    };
+    window.remotion_broadcastChannel.postMessage(request);
+    let timeoutId;
+    return Promise.race([
+        resolvePromise.then((res) => {
+            clearTimeout(timeoutId);
+            return res;
+        }),
+        new Promise((_, reject) => {
+            timeoutId = setTimeout(() => {
+                reject(new Error(`Timeout while extracting frame at time ${timeInSeconds}sec from ${src}`));
+            }, Math.max(3000, window.remotion_puppeteerTimeout - 5000));
+        }),
+    ]);
+};

package/dist/video-extraction/extract-frame.d.ts

@@ -0,0 +1,9 @@
+import type { LogLevel } from '../log';
+import { type GetSink } from './get-frames-since-keyframe';
+export declare const sinkPromises: Record<string, Promise<GetSink>>;
+export declare const extractFrame: ({ src, timeInSeconds: unloopedTimeinSeconds, logLevel, loop, }: {
+    src: string;
+    timeInSeconds: number;
+    logLevel: LogLevel;
+    loop: boolean;
+}) => Promise<import("mediabunny").VideoSample | null>;

package/dist/video-extraction/extract-frame.js

@@ -0,0 +1,21 @@
+import { keyframeManager } from '../caches';
+import { getSinks } from './get-frames-since-keyframe';
+export const sinkPromises = {};
+export const extractFrame = async ({ src, timeInSeconds: unloopedTimeinSeconds, logLevel, loop, }) => {
+    if (!sinkPromises[src]) {
+        sinkPromises[src] = getSinks(src);
+    }
+    const { video, getDuration } = await sinkPromises[src];
+    const timeInSeconds = loop
+        ? unloopedTimeinSeconds % (await getDuration())
+        : unloopedTimeinSeconds;
+    const keyframeBank = await keyframeManager.requestKeyframeBank({
+        packetSink: video.packetSink,
+        videoSampleSink: video.sampleSink,
+        timestamp: timeInSeconds,
+        src,
+        logLevel,
+    });
+    const frame = await keyframeBank.getFrameFromTimestamp(timeInSeconds);
+    return frame;
+};

package/dist/video-extraction/get-frames-since-keyframe.d.ts

@@ -0,0 +1,23 @@
+import type { EncodedPacket } from 'mediabunny';
+import { AudioSampleSink, EncodedPacketSink, VideoSampleSink } from 'mediabunny';
+export declare const getSinks: (src: string) => Promise<{
+    video: {
+        sampleSink: VideoSampleSink;
+        packetSink: EncodedPacketSink;
+    };
+    audio: {
+        sampleSink: AudioSampleSink;
+    } | null;
+    actualMatroskaTimestamps: {
+        observeTimestamp: (startTime: number) => void;
+        getRealTimestamp: (observedTimestamp: number) => number | null;
+    };
+    isMatroska: boolean;
+    getDuration: () => Promise<number>;
+}>;
+export type GetSink = Awaited<ReturnType<typeof getSinks>>;
+export declare const getFramesSinceKeyframe: ({ packetSink, videoSampleSink, startPacket, }: {
+    packetSink: EncodedPacketSink;
+    videoSampleSink: VideoSampleSink;
+    startPacket: EncodedPacket;
+}) => Promise<import("./keyframe-bank").KeyframeBank>;

package/dist/video-extraction/get-frames-since-keyframe.js

@@ -0,0 +1,42 @@
+import { ALL_FORMATS, AudioSampleSink, EncodedPacketSink, Input, MATROSKA, UrlSource, VideoSampleSink, } from 'mediabunny';
+import { makeKeyframeBank } from './keyframe-bank';
+import { rememberActualMatroskaTimestamps } from './remember-actual-matroska-timestamps';
+export const getSinks = async (src) => {
+    const input = new Input({
+        formats: ALL_FORMATS,
+        source: new UrlSource(src),
+    });
+    const format = await input.getFormat();
+    const videoTrack = await input.getPrimaryVideoTrack();
+    if (!videoTrack) {
+        throw new Error(`No video track found for ${src}`);
+    }
+    const audioTrack = await input.getPrimaryAudioTrack();
+    const isMatroska = format === MATROSKA;
+    return {
+        video: {
+            sampleSink: new VideoSampleSink(videoTrack),
+            packetSink: new EncodedPacketSink(videoTrack),
+        },
+        audio: audioTrack
+            ? {
+                sampleSink: new AudioSampleSink(audioTrack),
+            }
+            : null,
+        actualMatroskaTimestamps: rememberActualMatroskaTimestamps(isMatroska),
+        isMatroska,
+        getDuration: () => input.computeDuration(),
+    };
+};
+export const getFramesSinceKeyframe = async ({ packetSink, videoSampleSink, startPacket, }) => {
+    const nextKeyPacket = await packetSink.getNextKeyPacket(startPacket, {
+        verifyKeyPackets: false,
+    });
+    const sampleIterator = videoSampleSink.samples(startPacket.timestamp, nextKeyPacket ? nextKeyPacket.timestamp : Infinity);
+    const keyframeBank = makeKeyframeBank({
+        startTimestampInSeconds: startPacket.timestamp,
+        endTimestampInSeconds: nextKeyPacket ? nextKeyPacket.timestamp : Infinity,
+        sampleIterator,
+    });
+    return keyframeBank;
+};

package/dist/video-extraction/keyframe-bank.d.ts

@@ -0,0 +1,25 @@
+import type { VideoSample } from 'mediabunny';
+import type { LogLevel } from '../log';
+export type KeyframeBank = {
+    startTimestampInSeconds: number;
+    endTimestampInSeconds: number;
+    getFrameFromTimestamp: (timestamp: number) => Promise<VideoSample | null>;
+    prepareForDeletion: () => Promise<void>;
+    deleteFramesBeforeTimestamp: ({ logLevel, src, timestampInSeconds, }: {
+        timestampInSeconds: number;
+        logLevel: LogLevel;
+        src: string;
+    }) => void;
+    hasTimestampInSecond: (timestamp: number) => Promise<boolean>;
+    addFrame: (frame: VideoSample) => void;
+    getOpenFrameCount: () => {
+        size: number;
+        timestamps: number[];
+    };
+    getLastUsed: () => number;
+};
+export declare const makeKeyframeBank: ({ startTimestampInSeconds, endTimestampInSeconds, sampleIterator, }: {
+    startTimestampInSeconds: number;
+    endTimestampInSeconds: number;
+    sampleIterator: AsyncGenerator<VideoSample, void, unknown>;
+}) => KeyframeBank;

package/dist/video-extraction/keyframe-bank.js

@@ -0,0 +1,121 @@
+import { Log } from '../log';
+// Round to only 4 digits, because WebM has a timescale of 1_000, e.g. framer.webm
+const roundTo4Digits = (timestamp) => {
+    return Math.round(timestamp * 1000) / 1000;
+};
+export const makeKeyframeBank = ({ startTimestampInSeconds, endTimestampInSeconds, sampleIterator, }) => {
+    const frames = {};
+    const frameTimestamps = [];
+    let lastUsed = Date.now();
+    let alloctionSize = 0;
+    const hasDecodedEnoughForTimestamp = (timestamp) => {
+        const lastFrameTimestamp = frameTimestamps[frameTimestamps.length - 1];
+        if (!lastFrameTimestamp) {
+            return false;
+        }
+        const lastFrame = frames[lastFrameTimestamp];
+        // Don't decode more, will probably have to re-decode everything
+        if (!lastFrame) {
+            return true;
+        }
+        return (roundTo4Digits(lastFrame.timestamp + lastFrame.duration) >
+            roundTo4Digits(timestamp));
+    };
+    const addFrame = (frame) => {
+        frames[frame.timestamp] = frame;
+        frameTimestamps.push(frame.timestamp);
+        alloctionSize += frame.allocationSize();
+        lastUsed = Date.now();
+    };
+    const ensureEnoughFramesForTimestamp = async (timestamp) => {
+        while (!hasDecodedEnoughForTimestamp(timestamp)) {
+            const sample = await sampleIterator.next();
+            if (sample.value) {
+                addFrame(sample.value);
+            }
+            if (sample.done) {
+                break;
+            }
+        }
+        lastUsed = Date.now();
+    };
+    const getFrameFromTimestamp = async (timestampInSeconds) => {
+        lastUsed = Date.now();
+        if (timestampInSeconds < startTimestampInSeconds) {
+            return Promise.reject(new Error(`Timestamp is before start timestamp (requested: ${timestampInSeconds}sec, start: ${startTimestampInSeconds})`));
+        }
+        if (timestampInSeconds > endTimestampInSeconds) {
+            return Promise.reject(new Error(`Timestamp is after end timestamp (requested: ${timestampInSeconds}sec, end: ${endTimestampInSeconds})`));
+        }
+        await ensureEnoughFramesForTimestamp(timestampInSeconds);
+        for (let i = frameTimestamps.length - 1; i >= 0; i--) {
+            const sample = frames[frameTimestamps[i]];
+            if (!sample) {
+                return null;
+            }
+            if (roundTo4Digits(sample.timestamp) <= roundTo4Digits(timestampInSeconds)) {
+                return sample;
+            }
+        }
+        return null;
+    };
+    const hasTimestampInSecond = async (timestamp) => {
+        return (await getFrameFromTimestamp(timestamp)) !== null;
+    };
+    const prepareForDeletion = async () => {
+        // Cleanup frames that have been extracted that might not have been retrieved yet
+        const { value } = await sampleIterator.return();
+        if (value) {
+            value.close();
+        }
+        for (const frameTimestamp of frameTimestamps) {
+            if (!frames[frameTimestamp]) {
+                continue;
+            }
+            alloctionSize -= frames[frameTimestamp].allocationSize();
+            frames[frameTimestamp].close();
+            delete frames[frameTimestamp];
+        }
+        frameTimestamps.length = 0;
+    };
+    const deleteFramesBeforeTimestamp = ({ logLevel, src, timestampInSeconds, }) => {
+        for (const frameTimestamp of frameTimestamps) {
+            const isLast = frameTimestamp === frameTimestamps[frameTimestamps.length - 1];
+            // Don't delete the last frame, since it may be the last one in the video!
+            if (isLast) {
+                continue;
+            }
+            if (frameTimestamp < timestampInSeconds) {
+                if (!frames[frameTimestamp]) {
+                    continue;
+                }
+                alloctionSize -= frames[frameTimestamp].allocationSize();
+                frameTimestamps.splice(frameTimestamps.indexOf(frameTimestamp), 1);
+                frames[frameTimestamp].close();
+                delete frames[frameTimestamp];
+                Log.verbose(logLevel, `[Video] Deleted frame ${frameTimestamp} for src ${src}`);
+            }
+        }
+    };
+    const getOpenFrameCount = () => {
+        return {
+            size: alloctionSize,
+            timestamps: frameTimestamps,
+        };
+    };
+    const getLastUsed = () => {
+        return lastUsed;
+    };
+    const keyframeBank = {
+        startTimestampInSeconds,
+        endTimestampInSeconds,
+        getFrameFromTimestamp,
+        prepareForDeletion,
+        hasTimestampInSecond,
+        addFrame,
+        deleteFramesBeforeTimestamp,
+        getOpenFrameCount,
+        getLastUsed,
+    };
+    return keyframeBank;
+};

package/dist/video-extraction/keyframe-manager.d.ts

@@ -0,0 +1,23 @@
+import type { EncodedPacketSink, VideoSampleSink } from 'mediabunny';
+import type { LogLevel } from '../log';
+import { type KeyframeBank } from './keyframe-bank';
+export declare const makeKeyframeManager: () => {
+    requestKeyframeBank: ({ packetSink, timestamp, videoSampleSink, src, logLevel, }: {
+        timestamp: number;
+        packetSink: EncodedPacketSink;
+        videoSampleSink: VideoSampleSink;
+        src: string;
+        logLevel: LogLevel;
+    }) => Promise<KeyframeBank>;
+    addKeyframeBank: ({ src, bank, startTimestampInSeconds, }: {
+        src: string;
+        bank: Promise<KeyframeBank>;
+        startTimestampInSeconds: number;
+    }) => void;
+    getCacheStats: () => Promise<{
+        count: number;
+        totalSize: number;
+    }>;
+    clearAll: () => Promise<void>;
+};
+export type KeyframeManager = Awaited<ReturnType<typeof makeKeyframeManager>>;

package/dist/video-extraction/keyframe-manager.js

@@ -0,0 +1,171 @@
+import { getTotalCacheStats, MAX_CACHE_SIZE, SAFE_BACK_WINDOW_IN_SECONDS, } from '../caches';
+import { Log } from '../log';
+import { getFramesSinceKeyframe } from './get-frames-since-keyframe';
+export const makeKeyframeManager = () => {
+    // src => {[startTimestampInSeconds]: KeyframeBank
+    const sources = {};
+    const addKeyframeBank = ({ src, bank, startTimestampInSeconds, }) => {
+        sources[src] = sources[src] ?? {};
+        sources[src][startTimestampInSeconds] = bank;
+    };
+    const logCacheStats = async (logLevel) => {
+        let count = 0;
+        let totalSize = 0;
+        for (const src in sources) {
+            for (const bank in sources[src]) {
+                const v = await sources[src][bank];
+                const { size, timestamps } = v.getOpenFrameCount();
+                count += timestamps.length;
+                totalSize += size;
+                if (size === 0) {
+                    continue;
+                }
+                Log.verbose(logLevel, `[Video] Open frames for src ${src}: ${timestamps.join(', ')}`);
+            }
+        }
+        Log.verbose(logLevel, `[Video] Cache stats: ${count} open frames, ${totalSize} bytes`);
+    };
+    const getCacheStats = async () => {
+        let count = 0;
+        let totalSize = 0;
+        for (const src in sources) {
+            for (const bank in sources[src]) {
+                const v = await sources[src][bank];
+                const { timestamps, size } = v.getOpenFrameCount();
+                count += timestamps.length;
+                totalSize += size;
+                if (size === 0) {
+                    continue;
+                }
+            }
+        }
+        return { count, totalSize };
+    };
+    const getTheKeyframeBankMostInThePast = async () => {
+        let mostInThePast = null;
+        let mostInThePastBank = null;
+        for (const src in sources) {
+            for (const b in sources[src]) {
+                const bank = await sources[src][b];
+                const lastUsed = bank.getLastUsed();
+                if (mostInThePast === null || lastUsed < mostInThePast) {
+                    mostInThePast = lastUsed;
+                    mostInThePastBank = { src, bank };
+                }
+            }
+        }
+        if (!mostInThePastBank) {
+            throw new Error('No keyframe bank found');
+        }
+        return mostInThePastBank;
+    };
+    const deleteOldestKeyframeBank = async (logLevel) => {
+        const { bank: mostInThePastBank, src: mostInThePastSrc } = await getTheKeyframeBankMostInThePast();
+        if (mostInThePastBank) {
+            await mostInThePastBank.prepareForDeletion();
+            delete sources[mostInThePastSrc][mostInThePastBank.startTimestampInSeconds];
+            Log.verbose(logLevel, `[Video] Deleted frames for src ${mostInThePastSrc} from ${mostInThePastBank.startTimestampInSeconds}sec to ${mostInThePastBank.endTimestampInSeconds}sec to free up memory.`);
+        }
+    };
+    const ensureToStayUnderMaxCacheSize = async (logLevel) => {
+        let cacheStats = await getTotalCacheStats();
+        while (cacheStats.totalSize > MAX_CACHE_SIZE) {
+            await deleteOldestKeyframeBank(logLevel);
+            cacheStats = await getTotalCacheStats();
+        }
+    };
+    const clearKeyframeBanksBeforeTime = async ({ timestampInSeconds, src, logLevel, }) => {
+        const threshold = timestampInSeconds - SAFE_BACK_WINDOW_IN_SECONDS;
+        if (!sources[src]) {
+            return;
+        }
+        const banks = Object.keys(sources[src]);
+        for (const startTimeInSeconds of banks) {
+            const bank = await sources[src][startTimeInSeconds];
+            const { endTimestampInSeconds, startTimestampInSeconds } = bank;
+            if (endTimestampInSeconds < threshold) {
+                await bank.prepareForDeletion();
+                Log.verbose(logLevel, `[Video] Cleared frames for src ${src} from ${startTimestampInSeconds}sec to ${endTimestampInSeconds}sec`);
+                delete sources[src][startTimeInSeconds];
+            }
+            else {
+                bank.deleteFramesBeforeTimestamp({
+                    timestampInSeconds: threshold,
+                    logLevel,
+                    src,
+                });
+            }
+        }
+        await logCacheStats(logLevel);
+    };
+    const getKeyframeBankOrRefetch = async ({ packetSink, timestamp, videoSampleSink, src, logLevel, }) => {
+        const startPacket = await packetSink.getKeyPacket(timestamp, {
+            verifyKeyPackets: true,
+        });
+        if (!startPacket) {
+            throw new Error(`No key packet found for timestamp ${timestamp}`);
+        }
+        const startTimestampInSeconds = startPacket.timestamp;
+        const existingBank = sources[src]?.[startTimestampInSeconds];
+        // Bank does not yet exist, we need to fetch
+        if (!existingBank) {
+            const newKeyframeBank = getFramesSinceKeyframe({
+                packetSink,
+                videoSampleSink,
+                startPacket,
+            });
+            addKeyframeBank({ src, bank: newKeyframeBank, startTimestampInSeconds });
+            return newKeyframeBank;
+        }
+        // Bank exists and still has the frame we want
+        if (await (await existingBank).hasTimestampInSecond(timestamp)) {
+            return existingBank;
+        }
+        Log.verbose(logLevel, `[Video] Bank exists but frames have already been evicted!`);
+        // Bank exists but frames have already been evicted!
+        // First delete it entirely
+        await (await existingBank).prepareForDeletion();
+        delete sources[src][startTimestampInSeconds];
+        // Then refetch
+        const replacementKeybank = getFramesSinceKeyframe({
+            packetSink,
+            videoSampleSink,
+            startPacket,
+        });
+        addKeyframeBank({ src, bank: replacementKeybank, startTimestampInSeconds });
+        return replacementKeybank;
+    };
+    const requestKeyframeBank = async ({ packetSink, timestamp, videoSampleSink, src, logLevel, }) => {
+        await ensureToStayUnderMaxCacheSize(logLevel);
+        await clearKeyframeBanksBeforeTime({
+            timestampInSeconds: timestamp,
+            src,
+            logLevel,
+        });
+        const keyframeBank = await getKeyframeBankOrRefetch({
+            packetSink,
+            timestamp,
+            videoSampleSink,
+            src,
+            logLevel,
+        });
+        return keyframeBank;
+    };
+    const clearAll = async () => {
+        const srcs = Object.keys(sources);
+        for (const src of srcs) {
+            const banks = Object.keys(sources[src]);
+            for (const startTimeInSeconds of banks) {
+                const bank = await sources[src][startTimeInSeconds];
+                await bank.prepareForDeletion();
+                delete sources[src][startTimeInSeconds];
+            }
+        }
+    };
+    return {
+        requestKeyframeBank,
+        addKeyframeBank,
+        getCacheStats,
+        clearAll,
+    };
+};

package/dist/video-extraction/remember-actual-matroska-timestamps.d.ts

@@ -0,0 +1,5 @@
+export declare const rememberActualMatroskaTimestamps: (isMatroska: boolean) => {
+    observeTimestamp: (startTime: number) => void;
+    getRealTimestamp: (observedTimestamp: number) => number | null;
+};
+export type RememberActualMatroskaTimestamps = ReturnType<typeof rememberActualMatroskaTimestamps>;

package/dist/video-extraction/remember-actual-matroska-timestamps.js

@@ -0,0 +1,19 @@
+export const rememberActualMatroskaTimestamps = (isMatroska) => {
+    const observations = [];
+    const observeTimestamp = (startTime) => {
+        if (!isMatroska) {
+            return;
+        }
+        observations.push(startTime);
+    };
+    const getRealTimestamp = (observedTimestamp) => {
+        if (!isMatroska) {
+            return observedTimestamp;
+        }
+        return (observations.find((observation) => Math.abs(observedTimestamp - observation) < 0.001) ?? null);
+    };
+    return {
+        observeTimestamp,
+        getRealTimestamp,
+    };
+};

package/dist/video-for-rendering.d.ts

@@ -0,0 +1,3 @@
+import React from 'react';
+import type { VideoProps } from './video/props';
+export declare const VideoForRendering: React.FC<VideoProps>;