@remotion/media 4.0.428 → 4.0.430

package/dist/audio/audio-preview-iterator.js

@@ -0,0 +1,176 @@
+import { roundTo4Digits } from '../helpers/round-to-4-digits';
+import { allowWaitRoutine } from './allow-wait';
+export const HEALTHY_BUFFER_THRESHOLD_SECONDS = 1;
+export const makeAudioIterator = (startFromSecond, cache) => {
+    let destroyed = false;
+    const iterator = cache.makeIteratorOrUsePrewarmed(startFromSecond);
+    const queuedAudioNodes = [];
+    const audioChunksForAfterResuming = [];
+    let mostRecentTimestamp = -Infinity;
+    const cleanupAudioQueue = () => {
+        for (const node of queuedAudioNodes) {
+            node.node.stop();
+        }
+        queuedAudioNodes.length = 0;
+    };
+    const getNextOrNullIfNotAvailable = async (allowWait) => {
+        const next = iterator.next();
+        const result = allowWait
+            ? await allowWaitRoutine(next, allowWait)
+            : await Promise.race([
+                next,
+                new Promise((resolve) => {
+                    Promise.resolve().then(() => resolve());
+                }),
+            ]);
+        if (!result) {
+            return {
+                type: 'need-to-wait-for-it',
+                waitPromise: async () => {
+                    const res = await next;
+                    return res.value;
+                },
+            };
+        }
+        if (result.value) {
+            mostRecentTimestamp = Math.max(mostRecentTimestamp, result.value.timestamp + result.value.duration);
+            return {
+                type: 'got-buffer',
+                buffer: result.value,
+            };
+        }
+        return {
+            type: 'got-end',
+            mostRecentTimestamp,
+        };
+    };
+    const tryToSatisfySeek = async (time, allowWait, onBufferScheduled) => {
+        if (time < startFromSecond) {
+            return {
+                type: 'not-satisfied',
+                reason: `time requested is before the start of the iterator`,
+            };
+        }
+        while (true) {
+            const buffer = await getNextOrNullIfNotAvailable(allowWait);
+            if (buffer.type === 'need-to-wait-for-it') {
+                return {
+                    type: 'not-satisfied',
+                    reason: 'iterator did not have buffer ready',
+                };
+            }
+            if (buffer.type === 'got-end') {
+                if (time >= mostRecentTimestamp) {
+                    return {
+                        type: 'ended',
+                    };
+                }
+                return {
+                    type: 'not-satisfied',
+                    reason: `iterator ended before the requested time`,
+                };
+            }
+            if (buffer.type === 'got-buffer') {
+                const bufferTimestamp = roundTo4Digits(buffer.buffer.timestamp);
+                const bufferEndTimestamp = roundTo4Digits(buffer.buffer.timestamp + buffer.buffer.duration);
+                const timestamp = roundTo4Digits(time);
+                if (roundTo4Digits(time) < bufferTimestamp) {
+                    return {
+                        type: 'not-satisfied',
+                        reason: `iterator is too far, most recently returned ${bufferTimestamp}-${bufferEndTimestamp}, requested ${time}`,
+                    };
+                }
+                if (bufferTimestamp <= timestamp && bufferEndTimestamp > timestamp) {
+                    onBufferScheduled(buffer.buffer);
+                    return {
+                        type: 'satisfied',
+                    };
+                }
+                onBufferScheduled(buffer.buffer);
+                continue;
+            }
+            throw new Error('Unreachable');
+        }
+    };
+    const removeAndReturnAllQueuedAudioNodes = () => {
+        const nodes = queuedAudioNodes.slice();
+        for (const node of nodes) {
+            node.node.stop();
+        }
+        queuedAudioNodes.length = 0;
+        return nodes;
+    };
+    const addChunkForAfterResuming = (buffer, timestamp) => {
+        audioChunksForAfterResuming.push({ buffer, timestamp });
+    };
+    const moveQueuedChunksToPauseQueue = () => {
+        const toQueue = removeAndReturnAllQueuedAudioNodes();
+        for (const chunk of toQueue) {
+            addChunkForAfterResuming(chunk.buffer, chunk.timestamp);
+        }
+    };
+    const getNumberOfChunksAfterResuming = () => {
+        return audioChunksForAfterResuming.length;
+    };
+    return {
+        destroy: () => {
+            cleanupAudioQueue();
+            destroyed = true;
+            iterator.return().catch(() => undefined);
+            audioChunksForAfterResuming.length = 0;
+        },
+        getNext: async () => {
+            const next = await iterator.next();
+            if (next.value) {
+                mostRecentTimestamp = Math.max(mostRecentTimestamp, next.value.timestamp + next.value.duration);
+            }
+            return next;
+        },
+        isDestroyed: () => {
+            return destroyed;
+        },
+        addQueuedAudioNode: (node, timestamp, buffer) => {
+            queuedAudioNodes.push({ node, timestamp, buffer });
+        },
+        removeQueuedAudioNode: (node) => {
+            const index = queuedAudioNodes.findIndex((n) => n.node === node);
+            if (index !== -1) {
+                queuedAudioNodes.splice(index, 1);
+            }
+        },
+        getAndClearAudioChunksForAfterResuming: () => {
+            const chunks = audioChunksForAfterResuming.slice();
+            audioChunksForAfterResuming.length = 0;
+            return chunks;
+        },
+        getQueuedPeriod: () => {
+            let until = -Infinity;
+            let from = Infinity;
+            for (const node of queuedAudioNodes) {
+                until = Math.max(until, node.timestamp + node.buffer.duration);
+                from = Math.min(from, node.timestamp);
+            }
+            for (const chunk of audioChunksForAfterResuming) {
+                until = Math.max(until, chunk.timestamp + chunk.buffer.duration);
+                from = Math.min(from, chunk.timestamp);
+            }
+            if (!Number.isFinite(from) || !Number.isFinite(until)) {
+                return null;
+            }
+            return {
+                from,
+                until,
+            };
+        },
+        tryToSatisfySeek,
+        addChunkForAfterResuming,
+        moveQueuedChunksToPauseQueue,
+        getNumberOfChunksAfterResuming,
+    };
+};
+export const isAlreadyQueued = (time, queuedPeriod) => {
+    if (!queuedPeriod) {
+        return false;
+    }
+    return time >= queuedPeriod.from && time < queuedPeriod.until;
+};

package/dist/audio/audio.js

@@ -0,0 +1,20 @@
+import { jsx as _jsx } from "react/jsx-runtime";
+import { Internals, useRemotionEnvironment } from 'remotion';
+import { AudioForPreview } from './audio-for-preview';
+import { AudioForRendering } from './audio-for-rendering';
+const { validateMediaProps } = Internals;
+export const Audio = (props) => {
+    // Should only destruct `trimBefore` and `trimAfter` from props,
+    // rest gets drilled down
+    const { name, stack, showInTimeline, ...otherProps } = props;
+    const environment = useRemotionEnvironment();
+    if (typeof props.src !== 'string') {
+        throw new TypeError(`The \`<Audio>\` tag requires a string for \`src\`, but got ${JSON.stringify(props.src)} instead.`);
+    }
+    validateMediaProps({ playbackRate: props.playbackRate, volume: props.volume }, 'Audio');
+    if (environment.isRendering) {
+        return _jsx(AudioForRendering, { ...otherProps });
+    }
+    return _jsx(AudioForPreview, { name: name, ...otherProps, stack: stack ?? null });
+};
+Internals.addSequenceStackTraces(Audio);

package/dist/audio/props.js

@@ -0,0 +1 @@
+export {};

package/dist/audio-extraction/audio-cache.js

@@ -0,0 +1,66 @@
+export const makeAudioCache = () => {
+    const timestamps = [];
+    const samples = {};
+    const addFrame = (sample) => {
+        timestamps.push(sample.timestamp);
+        samples[sample.timestamp] = sample;
+    };
+    const clearBeforeThreshold = (threshold) => {
+        for (const timestamp of timestamps.slice()) {
+            const endTimestamp = timestamp + samples[timestamp].duration;
+            if (endTimestamp < threshold) {
+                const isLast = timestamp === timestamps[timestamps.length - 1];
+                if (isLast) {
+                    continue;
+                }
+                samples[timestamp].close();
+                delete samples[timestamp];
+                timestamps.splice(timestamps.indexOf(timestamp), 1);
+            }
+        }
+    };
+    const deleteAll = () => {
+        for (const timestamp of timestamps) {
+            samples[timestamp].close();
+            delete samples[timestamp];
+        }
+        timestamps.length = 0;
+    };
+    const getSamples = (timestamp, durationInSeconds) => {
+        const selected = [];
+        for (let i = 0; i < timestamps.length; i++) {
+            const sampleTimestamp = timestamps[i];
+            const sample = samples[sampleTimestamp];
+            if (sample.timestamp + sample.duration - 0.0000000001 <= timestamp) {
+                continue;
+            }
+            if (sample.timestamp >= timestamp + durationInSeconds - 0.0000000001) {
+                break;
+            }
+            selected.push(sample);
+        }
+        return selected;
+    };
+    const getOpenTimestamps = () => {
+        return timestamps;
+    };
+    const getOldestTimestamp = () => {
+        return timestamps[0];
+    };
+    const getNewestTimestamp = () => {
+        if (timestamps.length === 0) {
+            return null;
+        }
+        const sample = samples[timestamps[timestamps.length - 1]];
+        return sample.timestamp + sample.duration;
+    };
+    return {
+        addFrame,
+        clearBeforeThreshold,
+        deleteAll,
+        getSamples,
+        getOldestTimestamp,
+        getNewestTimestamp,
+        getOpenTimestamps,
+    };
+};

package/dist/audio-extraction/audio-iterator.js

@@ -0,0 +1,132 @@
+import { Internals } from 'remotion';
+import { SAFE_BACK_WINDOW_IN_SECONDS } from '../caches';
+import { makeAudioCache } from './audio-cache';
+// https://discord.com/channels/@me/1409810025844838481/1415028953093111870
+// Audio frames might have dependencies on previous and next frames so we need to decode a bit more
+// and then discard it.
+// The worst case seems to be FLAC files with a 65'535 sample window, which would be 1486.0ms at 44.1Khz.
+// So let's set a threshold of 1.5 seconds.
+const extraThreshold = 1.5;
+const warned = {};
+const warnAboutMatroskaOnce = (src, logLevel) => {
+    if (warned[src]) {
+        return;
+    }
+    warned[src] = true;
+    Internals.Log.warn({ logLevel, tag: '@remotion/media' }, `Audio from ${src} will need to be read from the beginning. https://www.remotion.dev/docs/media/support#matroska-limitation`);
+};
+export const makeAudioIterator = ({ audioSampleSink, isMatroska, startTimestamp, src, actualMatroskaTimestamps, logLevel, }) => {
+    // Matroska timestamps are not accurate unless we start from the beginning
+    // So for matroska, we need to decode all samples :(
+    // https://github.com/Vanilagy/mediabunny/issues/105
+    const sampleIterator = audioSampleSink.samples(isMatroska ? 0 : Math.max(0, startTimestamp - extraThreshold));
+    if (isMatroska) {
+        warnAboutMatroskaOnce(src, logLevel);
+    }
+    let fullDuration = null;
+    const cache = makeAudioCache();
+    let lastUsed = Date.now();
+    const getNextSample = async () => {
+        lastUsed = Date.now();
+        const { value: sample, done } = await sampleIterator.next();
+        if (done) {
+            fullDuration = cache.getNewestTimestamp();
+            return null;
+        }
+        const realTimestamp = actualMatroskaTimestamps.getRealTimestamp(sample.timestamp);
+        if (realTimestamp !== null && realTimestamp !== sample.timestamp) {
+            sample.setTimestamp(realTimestamp);
+        }
+        actualMatroskaTimestamps.observeTimestamp(sample.timestamp);
+        actualMatroskaTimestamps.observeTimestamp(sample.timestamp + sample.duration);
+        cache.addFrame(sample);
+        return sample;
+    };
+    const getSamples = async (timestamp, durationInSeconds) => {
+        lastUsed = Date.now();
+        if (fullDuration !== null && timestamp > fullDuration) {
+            // Clear all samples before the timestamp
+            // Do this in the while loop because samples might start from 0
+            cache.clearBeforeThreshold(fullDuration - SAFE_BACK_WINDOW_IN_SECONDS);
+            return [];
+        }
+        const samples = cache.getSamples(timestamp, durationInSeconds);
+        const newestTimestamp = cache.getNewestTimestamp();
+        if (newestTimestamp !== null) {
+            if (newestTimestamp >= timestamp + durationInSeconds - 0.0000000001) {
+                return samples;
+            }
+        }
+        while (true) {
+            const sample = await getNextSample();
+            // Clear all samples before the timestamp
+            // Do this in the while loop because samples might start from 0
+            // Also do this after a sample has just been added, if it was the last sample we now have the duration
+            // and can prevent deleting the last sample
+            const deleteBefore = fullDuration === null ? timestamp : Math.min(timestamp, fullDuration);
+            cache.clearBeforeThreshold(deleteBefore - SAFE_BACK_WINDOW_IN_SECONDS);
+            if (sample === null) {
+                break;
+            }
+            if (sample.timestamp + sample.duration - 0.0000000001 <= timestamp) {
+                continue;
+            }
+            if (sample.timestamp >= timestamp + durationInSeconds - 0.0000000001) {
+                break;
+            }
+            samples.push(sample);
+        }
+        return samples;
+    };
+    const logOpenFrames = () => {
+        const openTimestamps = cache.getOpenTimestamps();
+        if (openTimestamps.length > 0) {
+            const first = openTimestamps[0];
+            const last = openTimestamps[openTimestamps.length - 1];
+            Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, 'Open audio samples for src', src, `${first.toFixed(3)}...${last.toFixed(3)}`);
+        }
+    };
+    const getCacheStats = () => {
+        return {
+            count: cache.getOpenTimestamps().length,
+            size: cache.getOpenTimestamps().reduce((acc, t) => acc + t, 0),
+        };
+    };
+    const canSatisfyRequestedTime = (timestamp) => {
+        const oldestTimestamp = cache.getOldestTimestamp() ?? startTimestamp;
+        if (fullDuration !== null && timestamp > fullDuration) {
+            return true;
+        }
+        return (oldestTimestamp < timestamp && Math.abs(oldestTimestamp - timestamp) < 10);
+    };
+    const prepareForDeletion = () => {
+        cache.deleteAll();
+        sampleIterator.return().then((value) => {
+            if (value.value) {
+                value.value.close();
+            }
+        });
+        fullDuration = null;
+    };
+    let op = Promise.resolve([]);
+    return {
+        src,
+        getSamples: (ts, dur) => {
+            op = op.then(() => getSamples(ts, dur));
+            return op;
+        },
+        waitForCompletion: async () => {
+            await op;
+            return true;
+        },
+        canSatisfyRequestedTime,
+        logOpenFrames,
+        getCacheStats,
+        getLastUsed: () => lastUsed,
+        prepareForDeletion,
+        startTimestamp,
+        clearBeforeThreshold: cache.clearBeforeThreshold,
+        getOldestTimestamp: cache.getOldestTimestamp,
+        getNewestTimestamp: cache.getNewestTimestamp,
+    };
+};

package/dist/audio-extraction/audio-manager.js

@@ -0,0 +1,113 @@
+import { Internals } from 'remotion';
+import { getTotalCacheStats } from '../caches';
+import { makeAudioIterator } from './audio-iterator';
+export const makeAudioManager = () => {
+    const iterators = [];
+    const makeIterator = ({ timeInSeconds, src, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }) => {
+        const iterator = makeAudioIterator({
+            audioSampleSink,
+            isMatroska,
+            startTimestamp: timeInSeconds,
+            src,
+            actualMatroskaTimestamps,
+            logLevel,
+        });
+        iterators.push(iterator);
+        return iterator;
+    };
+    const getIteratorMostInThePast = () => {
+        let mostInThePast = null;
+        let mostInThePastIterator = null;
+        for (const iterator of iterators) {
+            const lastUsed = iterator.getLastUsed();
+            if (mostInThePast === null || lastUsed < mostInThePast) {
+                mostInThePast = lastUsed;
+                mostInThePastIterator = iterator;
+            }
+        }
+        return mostInThePastIterator;
+    };
+    const deleteOldestIterator = () => {
+        const iterator = getIteratorMostInThePast();
+        if (iterator) {
+            iterator.prepareForDeletion();
+            iterators.splice(iterators.indexOf(iterator), 1);
+        }
+    };
+    const deleteDuplicateIterators = (logLevel) => {
+        const seenKeys = new Set();
+        for (let i = 0; i < iterators.length; i++) {
+            const iterator = iterators[i];
+            const key = `${iterator.src}-${iterator.getOldestTimestamp()}-${iterator.getNewestTimestamp()}`;
+            if (seenKeys.has(key)) {
+                iterator.prepareForDeletion();
+                iterators.splice(iterators.indexOf(iterator), 1);
+                Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, `Deleted duplicate iterator for ${iterator.src}`);
+            }
+            seenKeys.add(key);
+        }
+    };
+    const getIterator = async ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, maxCacheSize, }) => {
+        while ((await getTotalCacheStats()).totalSize > maxCacheSize) {
+            deleteOldestIterator();
+        }
+        for (const iterator of iterators) {
+            if (iterator.src === src &&
+                (await iterator.waitForCompletion()) &&
+                iterator.canSatisfyRequestedTime(timeInSeconds)) {
+                return iterator;
+            }
+        }
+        for (let i = 0; i < iterators.length; i++) {
+            const iterator = iterators[i];
+            // delete iterator with same starting timestamp as requested
+            if (iterator.src === src && iterator.startTimestamp === timeInSeconds) {
+                iterator.prepareForDeletion();
+                iterators.splice(iterators.indexOf(iterator), 1);
+            }
+        }
+        deleteDuplicateIterators(logLevel);
+        return makeIterator({
+            src,
+            timeInSeconds,
+            audioSampleSink,
+            isMatroska,
+            actualMatroskaTimestamps,
+            logLevel,
+        });
+    };
+    const getCacheStats = () => {
+        let totalCount = 0;
+        let totalSize = 0;
+        for (const iterator of iterators) {
+            const { count, size } = iterator.getCacheStats();
+            totalCount += count;
+            totalSize += size;
+        }
+        return { count: totalCount, totalSize };
+    };
+    const logOpenFrames = () => {
+        for (const iterator of iterators) {
+            iterator.logOpenFrames();
+        }
+    };
+    let queue = Promise.resolve(undefined);
+    return {
+        getIterator: ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, maxCacheSize, }) => {
+            queue = queue.then(() => getIterator({
+                src,
+                timeInSeconds,
+                audioSampleSink,
+                isMatroska,
+                actualMatroskaTimestamps,
+                logLevel,
+                maxCacheSize,
+            }));
+            return queue;
+        },
+        getCacheStats,
+        getIteratorMostInThePast,
+        logOpenFrames,
+        deleteDuplicateIterators,
+    };
+};

package/dist/audio-extraction/extract-audio.js

@@ -0,0 +1,132 @@
+import { audioManager } from '../caches';
+import { combineAudioDataAndClosePrevious } from '../convert-audiodata/combine-audiodata';
+import { convertAudioData, fixFloatingPoint, } from '../convert-audiodata/convert-audiodata';
+import { TARGET_NUMBER_OF_CHANNELS, TARGET_SAMPLE_RATE, } from '../convert-audiodata/resample-audiodata';
+import { getSink } from '../get-sink';
+import { getTimeInSeconds } from '../get-time-in-seconds';
+import { isNetworkError, isUnsupportedConfigurationError, } from '../is-type-of-error';
+const extractAudioInternal = async ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds: durationNotYetApplyingPlaybackRate, logLevel, loop, playbackRate, audioStreamIndex, trimBefore, trimAfter, fps, maxCacheSize, }) => {
+    const { getAudio, actualMatroskaTimestamps, isMatroska, getDuration } = await getSink(src, logLevel);
+    let mediaDurationInSeconds = null;
+    if (loop) {
+        mediaDurationInSeconds = await getDuration();
+    }
+    const audio = await getAudio(audioStreamIndex);
+    if (audio === 'network-error') {
+        return 'network-error';
+    }
+    if (audio === 'no-audio-track') {
+        return { data: null, durationInSeconds: null };
+    }
+    if (audio === 'cannot-decode-audio') {
+        return 'cannot-decode';
+    }
+    if (audio === 'unknown-container-format') {
+        return 'unknown-container-format';
+    }
+    const timeInSeconds = getTimeInSeconds({
+        loop,
+        mediaDurationInSeconds,
+        unloopedTimeInSeconds,
+        src,
+        trimAfter,
+        playbackRate,
+        trimBefore,
+        fps,
+        ifNoMediaDuration: 'fail',
+    });
+    if (timeInSeconds === null) {
+        return { data: null, durationInSeconds: mediaDurationInSeconds };
+    }
+    try {
+        const sampleIterator = await audioManager.getIterator({
+            src,
+            timeInSeconds,
+            audioSampleSink: audio.sampleSink,
+            isMatroska,
+            actualMatroskaTimestamps,
+            logLevel,
+            maxCacheSize,
+        });
+        const durationInSeconds = durationNotYetApplyingPlaybackRate * playbackRate;
+        const samples = await sampleIterator.getSamples(timeInSeconds, durationInSeconds);
+        audioManager.logOpenFrames();
+        const audioDataArray = [];
+        for (let i = 0; i < samples.length; i++) {
+            const sample = samples[i];
+            // Less than 1 sample would be included - we did not need it after all!
+            if (Math.abs(sample.timestamp - (timeInSeconds + durationInSeconds)) *
+                sample.sampleRate <
+                1) {
+                continue;
+            }
+            // Less than 1 sample would be included - we did not need it after all!
+            if (sample.timestamp + sample.duration <= timeInSeconds) {
+                continue;
+            }
+            const isFirstSample = i === 0;
+            const isLastSample = i === samples.length - 1;
+            const audioDataRaw = sample.toAudioData();
+            // amount of samples to shave from start and end
+            let trimStartInSeconds = 0;
+            let trimEndInSeconds = 0;
+            let leadingSilence = null;
+            if (isFirstSample) {
+                trimStartInSeconds = fixFloatingPoint(timeInSeconds - sample.timestamp);
+                if (trimStartInSeconds < 0) {
+                    const silenceFrames = Math.ceil(fixFloatingPoint(-trimStartInSeconds * TARGET_SAMPLE_RATE));
+                    leadingSilence = {
+                        data: new Int16Array(silenceFrames * TARGET_NUMBER_OF_CHANNELS),
+                        numberOfFrames: silenceFrames,
+                        timestamp: timeInSeconds * 1000000,
+                        durationInMicroSeconds: (silenceFrames / TARGET_SAMPLE_RATE) * 1000000,
+                    };
+                    trimStartInSeconds = 0;
+                }
+            }
+            if (isLastSample) {
+                trimEndInSeconds =
+                    // clamp to 0 in case the audio ends early
+                    Math.max(0, sample.timestamp +
+                        sample.duration -
+                        (timeInSeconds + durationInSeconds));
+            }
+            const audioData = convertAudioData({
+                audioData: audioDataRaw,
+                trimStartInSeconds,
+                trimEndInSeconds,
+                playbackRate,
+                audioDataTimestamp: sample.timestamp,
+                isLast: isLastSample,
+            });
+            audioDataRaw.close();
+            if (audioData.numberOfFrames === 0) {
+                continue;
+            }
+            if (leadingSilence) {
+                audioDataArray.push(leadingSilence);
+            }
+            audioDataArray.push(audioData);
+        }
+        if (audioDataArray.length === 0) {
+            return { data: null, durationInSeconds: mediaDurationInSeconds };
+        }
+        const combined = combineAudioDataAndClosePrevious(audioDataArray);
+        return { data: combined, durationInSeconds: mediaDurationInSeconds };
+    }
+    catch (err) {
+        const error = err;
+        if (isNetworkError(error)) {
+            return 'network-error';
+        }
+        if (isUnsupportedConfigurationError(error)) {
+            return 'cannot-decode';
+        }
+        throw err;
+    }
+};
+let queue = Promise.resolve(undefined);
+export const extractAudio = (params) => {
+    queue = queue.then(() => extractAudioInternal(params));
+    return queue;
+};