@remotion/media 4.0.428 → 4.0.430

package/dist/video-extraction/keyframe-bank.js

@@ -0,0 +1,159 @@
+import { Internals } from 'remotion';
+import { SAFE_BACK_WINDOW_IN_SECONDS } from '../caches';
+import { roundTo4Digits } from '../helpers/round-to-4-digits';
+import { renderTimestampRange } from '../render-timestamp-range';
+import { getAllocationSize } from './get-allocation-size';
+export const makeKeyframeBank = ({ startTimestampInSeconds, endTimestampInSeconds, sampleIterator, logLevel: parentLogLevel, src, }) => {
+    Internals.Log.verbose({ logLevel: parentLogLevel, tag: '@remotion/media' }, `Creating keyframe bank from ${startTimestampInSeconds}sec to ${endTimestampInSeconds}sec`);
+    const frames = {};
+    const frameTimestamps = [];
+    let lastUsed = Date.now();
+    let allocationSize = 0;
+    const deleteFramesBeforeTimestamp = ({ logLevel, timestampInSeconds, }) => {
+        const deletedTimestamps = [];
+        for (const frameTimestamp of frameTimestamps.slice()) {
+            const isLast = frameTimestamp === frameTimestamps[frameTimestamps.length - 1];
+            // Don't delete the last frame, since it may be the last one in the video!
+            if (isLast) {
+                continue;
+            }
+            if (frameTimestamp < timestampInSeconds) {
+                if (!frames[frameTimestamp]) {
+                    continue;
+                }
+                allocationSize -= getAllocationSize(frames[frameTimestamp]);
+                frameTimestamps.splice(frameTimestamps.indexOf(frameTimestamp), 1);
+                frames[frameTimestamp].close();
+                delete frames[frameTimestamp];
+                deletedTimestamps.push(frameTimestamp);
+            }
+        }
+        if (deletedTimestamps.length > 0) {
+            Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, `Deleted ${deletedTimestamps.length} frame${deletedTimestamps.length === 1 ? '' : 's'} ${renderTimestampRange(deletedTimestamps)} for src ${src} because it is lower than ${timestampInSeconds}. Remaining: ${renderTimestampRange(frameTimestamps)}`);
+        }
+    };
+    const hasDecodedEnoughForTimestamp = (timestamp) => {
+        const lastFrameTimestamp = frameTimestamps[frameTimestamps.length - 1];
+        if (!lastFrameTimestamp) {
+            return false;
+        }
+        const lastFrame = frames[lastFrameTimestamp];
+        // Don't decode more, will probably have to re-decode everything
+        if (!lastFrame) {
+            return true;
+        }
+        return (roundTo4Digits(lastFrame.timestamp + lastFrame.duration) >
+            roundTo4Digits(timestamp) + 0.001);
+    };
+    const addFrame = (frame) => {
+        if (frames[frame.timestamp]) {
+            allocationSize -= getAllocationSize(frames[frame.timestamp]);
+            frameTimestamps.splice(frameTimestamps.indexOf(frame.timestamp), 1);
+            frames[frame.timestamp].close();
+            delete frames[frame.timestamp];
+        }
+        frames[frame.timestamp] = frame;
+        frameTimestamps.push(frame.timestamp);
+        allocationSize += getAllocationSize(frame);
+        lastUsed = Date.now();
+    };
+    const ensureEnoughFramesForTimestamp = async (timestampInSeconds) => {
+        while (!hasDecodedEnoughForTimestamp(timestampInSeconds)) {
+            const sample = await sampleIterator.next();
+            if (sample.value) {
+                addFrame(sample.value);
+            }
+            if (sample.done) {
+                break;
+            }
+            deleteFramesBeforeTimestamp({
+                logLevel: parentLogLevel,
+                timestampInSeconds: timestampInSeconds - SAFE_BACK_WINDOW_IN_SECONDS,
+            });
+        }
+        lastUsed = Date.now();
+    };
+    const getFrameFromTimestamp = async (timestampInSeconds) => {
+        lastUsed = Date.now();
+        // If the requested timestamp is before the start of this bank, clamp it to the start.
+        // This matches Chrome's behavior: render the first available frame rather than showing black.
+        // Videos don't always start at timestamp 0 due to encoding artifacts, container format quirks,
+        // and keyframe positioning. Users have no control over this, so we clamp to the first frame.
+        // Test case: https://github.com/remotion-dev/remotion/issues/5915
+        let adjustedTimestamp = timestampInSeconds;
+        if (roundTo4Digits(timestampInSeconds) <
+            roundTo4Digits(startTimestampInSeconds)) {
+            adjustedTimestamp = startTimestampInSeconds;
+        }
+        // If we request a timestamp after the end of the video, return the last frame
+        // same behavior as <video>
+        if (roundTo4Digits(adjustedTimestamp) > roundTo4Digits(endTimestampInSeconds)) {
+            adjustedTimestamp = endTimestampInSeconds;
+        }
+        await ensureEnoughFramesForTimestamp(adjustedTimestamp);
+        for (let i = frameTimestamps.length - 1; i >= 0; i--) {
+            const sample = frames[frameTimestamps[i]];
+            if (!sample) {
+                return null;
+            }
+            if (roundTo4Digits(sample.timestamp) <= roundTo4Digits(adjustedTimestamp) ||
+                // Match 0.3333333333 to 0.33355555
+                // this does not satisfy the previous condition, since one rounds up and one rounds down
+                Math.abs(sample.timestamp - adjustedTimestamp) <= 0.001) {
+                return sample;
+            }
+        }
+        return null;
+    };
+    const hasTimestampInSecond = async (timestamp) => {
+        return (await getFrameFromTimestamp(timestamp)) !== null;
+    };
+    const prepareForDeletion = (logLevel) => {
+        Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, `Preparing for deletion of keyframe bank from ${startTimestampInSeconds}sec to ${endTimestampInSeconds}sec`);
+        // Cleanup frames that have been extracted that might not have been retrieved yet
+        sampleIterator.return().then((result) => {
+            if (result.value) {
+                result.value.close();
+            }
+            return null;
+        });
+        let framesDeleted = 0;
+        for (const frameTimestamp of frameTimestamps) {
+            if (!frames[frameTimestamp]) {
+                continue;
+            }
+            allocationSize -= getAllocationSize(frames[frameTimestamp]);
+            frames[frameTimestamp].close();
+            delete frames[frameTimestamp];
+            framesDeleted++;
+        }
+        frameTimestamps.length = 0;
+        return { framesDeleted };
+    };
+    const getOpenFrameCount = () => {
+        return {
+            size: allocationSize,
+            timestamps: frameTimestamps,
+        };
+    };
+    const getLastUsed = () => {
+        return lastUsed;
+    };
+    let queue = Promise.resolve(undefined);
+    const keyframeBank = {
+        startTimestampInSeconds,
+        endTimestampInSeconds,
+        getFrameFromTimestamp: (timestamp) => {
+            queue = queue.then(() => getFrameFromTimestamp(timestamp));
+            return queue;
+        },
+        prepareForDeletion,
+        hasTimestampInSecond,
+        addFrame,
+        deleteFramesBeforeTimestamp,
+        src,
+        getOpenFrameCount,
+        getLastUsed,
+    };
+    return keyframeBank;
+};

package/dist/video-extraction/keyframe-manager.js

@@ -0,0 +1,206 @@
+import { Internals } from 'remotion';
+import { canBrowserUseWebGl2 } from '../browser-can-use-webgl2';
+import { getTotalCacheStats, SAFE_BACK_WINDOW_IN_SECONDS } from '../caches';
+import { renderTimestampRange } from '../render-timestamp-range';
+import { getFramesSinceKeyframe } from './get-frames-since-keyframe';
+export const makeKeyframeManager = () => {
+    // src => {[startTimestampInSeconds]: KeyframeBank
+    const sources = {};
+    const addKeyframeBank = ({ src, bank, startTimestampInSeconds, }) => {
+        sources[src] = sources[src] ?? {};
+        sources[src][startTimestampInSeconds] = bank;
+    };
+    const logCacheStats = async (logLevel) => {
+        let count = 0;
+        let totalSize = 0;
+        for (const src in sources) {
+            for (const bank in sources[src]) {
+                const v = await sources[src][bank];
+                const { size, timestamps } = v.getOpenFrameCount();
+                count += timestamps.length;
+                totalSize += size;
+                if (size === 0) {
+                    continue;
+                }
+                Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, `Open frames for src ${src}: ${renderTimestampRange(timestamps)}`);
+            }
+        }
+        Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, `Video cache stats: ${count} open frames, ${totalSize} bytes`);
+    };
+    const getCacheStats = async () => {
+        let count = 0;
+        let totalSize = 0;
+        for (const src in sources) {
+            for (const bank in sources[src]) {
+                const v = await sources[src][bank];
+                const { timestamps, size } = v.getOpenFrameCount();
+                count += timestamps.length;
+                totalSize += size;
+                if (size === 0) {
+                    continue;
+                }
+            }
+        }
+        return { count, totalSize };
+    };
+    const getTheKeyframeBankMostInThePast = async () => {
+        let mostInThePast = null;
+        let mostInThePastBank = null;
+        let numberOfBanks = 0;
+        for (const src in sources) {
+            for (const b in sources[src]) {
+                const bank = await sources[src][b];
+                const lastUsed = bank.getLastUsed();
+                if (mostInThePast === null || lastUsed < mostInThePast) {
+                    mostInThePast = lastUsed;
+                    mostInThePastBank = { src, bank };
+                }
+                numberOfBanks++;
+            }
+        }
+        if (!mostInThePastBank) {
+            throw new Error('No keyframe bank found');
+        }
+        return { mostInThePastBank, numberOfBanks };
+    };
+    const deleteOldestKeyframeBank = async (logLevel) => {
+        const { mostInThePastBank: { bank: mostInThePastBank, src: mostInThePastSrc }, numberOfBanks, } = await getTheKeyframeBankMostInThePast();
+        if (numberOfBanks < 2) {
+            return { finish: true };
+        }
+        if (mostInThePastBank) {
+            const { framesDeleted } = mostInThePastBank.prepareForDeletion(logLevel);
+            delete sources[mostInThePastSrc][mostInThePastBank.startTimestampInSeconds];
+            Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, `Deleted ${framesDeleted} frames for src ${mostInThePastSrc} from ${mostInThePastBank.startTimestampInSeconds}sec to ${mostInThePastBank.endTimestampInSeconds}sec to free up memory.`);
+        }
+        return { finish: false };
+    };
+    const ensureToStayUnderMaxCacheSize = async (logLevel, maxCacheSize) => {
+        let cacheStats = await getTotalCacheStats();
+        while (cacheStats.totalSize > maxCacheSize) {
+            const { finish } = await deleteOldestKeyframeBank(logLevel);
+            if (finish) {
+                break;
+            }
+            Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, 'Deleted oldest keyframe bank to stay under max cache size', (cacheStats.totalSize / 1024 / 1024).toFixed(1), 'out of', (maxCacheSize / 1024 / 1024).toFixed(1));
+            cacheStats = await getTotalCacheStats();
+        }
+    };
+    const clearKeyframeBanksBeforeTime = async ({ timestampInSeconds, src, logLevel, }) => {
+        const threshold = timestampInSeconds - SAFE_BACK_WINDOW_IN_SECONDS;
+        if (!sources[src]) {
+            return;
+        }
+        const banks = Object.keys(sources[src]);
+        for (const startTimeInSeconds of banks) {
+            const bank = await sources[src][startTimeInSeconds];
+            const { endTimestampInSeconds, startTimestampInSeconds } = bank;
+            if (endTimestampInSeconds < threshold) {
+                bank.prepareForDeletion(logLevel);
+                Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, `[Video] Cleared frames for src ${src} from ${startTimestampInSeconds}sec to ${endTimestampInSeconds}sec`);
+                delete sources[src][startTimeInSeconds];
+            }
+            else {
+                bank.deleteFramesBeforeTimestamp({
+                    timestampInSeconds: threshold,
+                    logLevel,
+                });
+            }
+        }
+        await logCacheStats(logLevel);
+    };
+    const getKeyframeBankOrRefetch = async ({ packetSink, timestamp, videoSampleSink, src, logLevel, }) => {
+        // Try to get the keypacket at the requested timestamp.
+        // If it returns null (timestamp is before the first keypacket), fall back to the first packet.
+        // This matches mediabunny's internal behavior and handles videos that don't start at timestamp 0.
+        const startPacket = (await packetSink.getKeyPacket(timestamp, {
+            verifyKeyPackets: true,
+        })) ?? (await packetSink.getFirstPacket({ verifyKeyPackets: true }));
+        const hasAlpha = startPacket?.sideData.alpha;
+        if (hasAlpha && !canBrowserUseWebGl2()) {
+            return 'has-alpha';
+        }
+        if (!startPacket) {
+            // e.g. https://discord.com/channels/809501355504959528/809501355504959531/1424400511070765086
+            // The video has an offset and the first frame is at time 0.033sec
+            // we shall not crash here but handle it gracefully
+            return null;
+        }
+        const startTimestampInSeconds = startPacket.timestamp;
+        const existingBank = sources[src]?.[startTimestampInSeconds];
+        // Bank does not yet exist, we need to fetch
+        if (!existingBank) {
+            const newKeyframeBank = getFramesSinceKeyframe({
+                packetSink,
+                videoSampleSink,
+                startPacket,
+                logLevel,
+                src,
+            });
+            addKeyframeBank({ src, bank: newKeyframeBank, startTimestampInSeconds });
+            return newKeyframeBank;
+        }
+        // Bank exists and still has the frame we want
+        if (await (await existingBank).hasTimestampInSecond(timestamp)) {
+            return existingBank;
+        }
+        Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, `Keyframe bank exists but frame at time ${timestamp} does not exist anymore.`);
+        // Bank exists but frames have already been evicted!
+        // First delete it entirely
+        await (await existingBank).prepareForDeletion(logLevel);
+        delete sources[src][startTimestampInSeconds];
+        // Then refetch
+        const replacementKeybank = getFramesSinceKeyframe({
+            packetSink,
+            videoSampleSink,
+            startPacket,
+            logLevel,
+            src,
+        });
+        addKeyframeBank({ src, bank: replacementKeybank, startTimestampInSeconds });
+        return replacementKeybank;
+    };
+    const requestKeyframeBank = async ({ packetSink, timestamp, videoSampleSink, src, logLevel, maxCacheSize, }) => {
+        await ensureToStayUnderMaxCacheSize(logLevel, maxCacheSize);
+        await clearKeyframeBanksBeforeTime({
+            timestampInSeconds: timestamp,
+            src,
+            logLevel,
+        });
+        const keyframeBank = await getKeyframeBankOrRefetch({
+            packetSink,
+            timestamp,
+            videoSampleSink,
+            src,
+            logLevel,
+        });
+        return keyframeBank;
+    };
+    const clearAll = async (logLevel) => {
+        const srcs = Object.keys(sources);
+        for (const src of srcs) {
+            const banks = Object.keys(sources[src]);
+            for (const startTimeInSeconds of banks) {
+                const bank = await sources[src][startTimeInSeconds];
+                bank.prepareForDeletion(logLevel);
+                delete sources[src][startTimeInSeconds];
+            }
+        }
+    };
+    let queue = Promise.resolve(undefined);
+    return {
+        requestKeyframeBank: ({ packetSink, timestamp, videoSampleSink, src, logLevel, maxCacheSize, }) => {
+            queue = queue.then(() => requestKeyframeBank({
+                packetSink,
+                timestamp,
+                videoSampleSink,
+                src,
+                logLevel,
+                maxCacheSize,
+            }));
+            return queue;
+        },
+        getCacheStats,
+        clearAll,
+    };
+};

package/dist/video-extraction/remember-actual-matroska-timestamps.js

@@ -0,0 +1,19 @@
+export const rememberActualMatroskaTimestamps = (isMatroska) => {
+    const observations = [];
+    const observeTimestamp = (startTime) => {
+        if (!isMatroska) {
+            return;
+        }
+        observations.push(startTime);
+    };
+    const getRealTimestamp = (observedTimestamp) => {
+        if (!isMatroska) {
+            return observedTimestamp;
+        }
+        return (observations.find((observation) => Math.abs(observedTimestamp - observation) < 0.001) ?? null);
+    };
+    return {
+        observeTimestamp,
+        getRealTimestamp,
+    };
+};

package/dist/video-extraction/rotate-frame.js

@@ -0,0 +1,34 @@
+export const rotateFrame = async ({ frame, rotation, }) => {
+    if (rotation === 0) {
+        const directBitmap = await createImageBitmap(frame);
+        frame.close();
+        return directBitmap;
+    }
+    const width = rotation === 90 || rotation === 270
+        ? frame.displayHeight
+        : frame.displayWidth;
+    const height = rotation === 90 || rotation === 270
+        ? frame.displayWidth
+        : frame.displayHeight;
+    const canvas = new OffscreenCanvas(width, height);
+    const ctx = canvas.getContext('2d');
+    if (!ctx) {
+        throw new Error('Could not get 2d context');
+    }
+    canvas.width = width;
+    canvas.height = height;
+    if (rotation === 90) {
+        ctx.translate(width, 0);
+    }
+    else if (rotation === 180) {
+        ctx.translate(width, height);
+    }
+    else if (rotation === 270) {
+        ctx.translate(0, height);
+    }
+    ctx.rotate(rotation * (Math.PI / 180));
+    ctx.drawImage(frame, 0, 0);
+    const bitmap = await createImageBitmap(canvas);
+    frame.close();
+    return bitmap;
+};

package/dist/video-iterator-manager.js

@@ -0,0 +1,109 @@
+import { CanvasSink } from 'mediabunny';
+import { Internals } from 'remotion';
+import { makePrewarmedVideoIteratorCache } from './prewarm-iterator-for-looping';
+import { createVideoIterator, } from './video/video-preview-iterator';
+export const videoIteratorManager = ({ delayPlaybackHandleIfNotPremounting, canvas, context, drawDebugOverlay, logLevel, getOnVideoFrameCallback, videoTrack, getEndTime, getStartTime, getIsLooping, }) => {
+    let videoIteratorsCreated = 0;
+    let videoFrameIterator = null;
+    let framesRendered = 0;
+    let currentDelayHandle = null;
+    if (canvas) {
+        canvas.width = videoTrack.displayWidth;
+        canvas.height = videoTrack.displayHeight;
+    }
+    const canvasSink = new CanvasSink(videoTrack, {
+        poolSize: 2,
+        fit: 'contain',
+        alpha: true,
+    });
+    const prewarmedVideoIteratorCache = makePrewarmedVideoIteratorCache(canvasSink);
+    const drawFrame = (frame) => {
+        if (context && canvas) {
+            context.clearRect(0, 0, canvas.width, canvas.height);
+            context.drawImage(frame.canvas, 0, 0);
+        }
+        framesRendered++;
+        drawDebugOverlay();
+        const callback = getOnVideoFrameCallback();
+        if (callback) {
+            callback(frame.canvas);
+        }
+        Internals.Log.trace({ logLevel, tag: '@remotion/media' }, `[MediaPlayer] Drew frame ${frame.timestamp.toFixed(3)}s`);
+    };
+    const startVideoIterator = async (timeToSeek, nonce) => {
+        videoFrameIterator?.destroy();
+        const iterator = createVideoIterator(timeToSeek, prewarmedVideoIteratorCache);
+        videoIteratorsCreated++;
+        videoFrameIterator = iterator;
+        const delayHandle = delayPlaybackHandleIfNotPremounting();
+        currentDelayHandle = delayHandle;
+        let frameResult;
+        try {
+            frameResult = await iterator.getNext();
+        }
+        finally {
+            delayHandle.unblock();
+            currentDelayHandle = null;
+        }
+        if (iterator.isDestroyed()) {
+            return;
+        }
+        if (nonce.isStale()) {
+            return;
+        }
+        if (videoFrameIterator.isDestroyed()) {
+            return;
+        }
+        if (!frameResult.value) {
+            // media ended
+            return;
+        }
+        drawFrame(frameResult.value);
+    };
+    const seek = async ({ newTime, nonce }) => {
+        if (!videoFrameIterator) {
+            return;
+        }
+        if (getIsLooping()) {
+            // If less than 1 second from the end away, we pre-warm a new iterator
+            if (getEndTime() - newTime < 1) {
+                prewarmedVideoIteratorCache.prewarmIteratorForLooping({
+                    timeToSeek: getStartTime(),
+                });
+            }
+        }
+        // Should return immediately, so it's okay to not use Promise.all here
+        const videoSatisfyResult = await videoFrameIterator.tryToSatisfySeek(newTime);
+        // Doing this before the staleness check, because
+        // frame might be better than what we currently have
+        // TODO: check if this is actually true
+        if (videoSatisfyResult.type === 'satisfied') {
+            drawFrame(videoSatisfyResult.frame);
+            return;
+        }
+        if (nonce.isStale()) {
+            return;
+        }
+        await startVideoIterator(newTime, nonce);
+    };
+    return {
+        startVideoIterator,
+        getVideoIteratorsCreated: () => videoIteratorsCreated,
+        seek,
+        destroy: () => {
+            prewarmedVideoIteratorCache.destroy();
+            videoFrameIterator?.destroy();
+            if (context && canvas) {
+                context.clearRect(0, 0, canvas.width, canvas.height);
+            }
+            if (currentDelayHandle) {
+                currentDelayHandle.unblock();
+                currentDelayHandle = null;
+            }
+            videoFrameIterator = null;
+        },
+        getVideoFrameIterator: () => videoFrameIterator,
+        drawFrame,
+        getFramesRendered: () => framesRendered,
+    };
+};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@remotion/media",
-	"version": "4.0.428",
+	"version": "4.0.430",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
 	"module": "dist/esm/index.mjs",
@@ -14,22 +14,24 @@
 	"type": "module",
 	"scripts": {
 		"if-node-18+": "node -e \"const [maj]=process.versions.node.split('.').map(Number); process.exit(maj>=18?0:1)\"",
-		"formatting": "prettier src --check",
+		"formatting": "oxfmt src --check",
+		"format": "oxfmt src",
 		"lint": "eslint src",
 		"watch": "tsgo -w",
 		"test": "node src/test/execute.mjs",
 		"make": "tsgo && bun --env-file=../.env.bundle bundle.ts"
 	},
 	"dependencies": {
-		"mediabunny": "1.34.4",
-		"remotion": "4.0.428"
+		"mediabunny": "1.35.1",
+		"remotion": "4.0.429",
+		"zod": "4.3.6"
 	},
 	"peerDependencies": {
 		"react": ">=16.8.0",
 		"react-dom": ">=16.8.0"
 	},
 	"devDependencies": {
-		"@remotion/eslint-config-internal": "4.0.428",
+		"@remotion/eslint-config-internal": "4.0.429",
 		"@vitest/browser-webdriverio": "4.0.9",
 		"eslint": "9.19.0",
 		"react": "19.2.3",