@remotion/media 4.0.356 → 4.0.358

package/dist/audio-extraction/audio-manager.d.ts

@@ -1,28 +1,8 @@
 import type { AudioSampleSink } from 'mediabunny';
-import type { LogLevel } from 'remotion';
+import { type LogLevel } from 'remotion';
 import type { RememberActualMatroskaTimestamps } from '../video-extraction/remember-actual-matroska-timestamps';
+import type { AudioSampleIterator } from './audio-iterator';
 export declare const makeAudioManager: () => {
-    makeIterator: ({ timeInSeconds, src, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }: {
-        timeInSeconds: number;
-        src: string;
-        audioSampleSink: AudioSampleSink;
-        isMatroska: boolean;
-        actualMatroskaTimestamps: RememberActualMatroskaTimestamps;
-        logLevel: LogLevel;
-    }) => {
-        src: string;
-        getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
-        waitForCompletion: () => Promise<boolean>;
-        canSatisfyRequestedTime: (timestamp: number) => boolean;
-        logOpenFrames: () => void;
-        getCacheStats: () => {
-            count: number;
-            size: number;
-        };
-        getLastUsed: () => number;
-        prepareForDeletion: () => Promise<void>;
-        startTimestamp: number;
-    };
     getIterator: ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }: {
         src: string;
         timeInSeconds: number;
@@ -30,20 +10,7 @@ export declare const makeAudioManager: () => {
         isMatroska: boolean;
         actualMatroskaTimestamps: RememberActualMatroskaTimestamps;
         logLevel: LogLevel;
-    }) => Promise<{
-        src: string;
-        getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
-        waitForCompletion: () => Promise<boolean>;
-        canSatisfyRequestedTime: (timestamp: number) => boolean;
-        logOpenFrames: () => void;
-        getCacheStats: () => {
-            count: number;
-            size: number;
-        };
-        getLastUsed: () => number;
-        prepareForDeletion: () => Promise<void>;
-        startTimestamp: number;
-    }>;
+    }) => Promise<AudioSampleIterator>;
     getCacheStats: () => {
         count: number;
         totalSize: number;
@@ -59,8 +26,12 @@ export declare const makeAudioManager: () => {
             size: number;
         };
         getLastUsed: () => number;
-        prepareForDeletion: () => Promise<void>;
+        prepareForDeletion: () => void;
         startTimestamp: number;
+        clearBeforeThreshold: (threshold: number) => void;
+        getOldestTimestamp: () => number;
+        getNewestTimestamp: () => number | null;
     } | null;
     logOpenFrames: () => void;
+    deleteDuplicateIterators: (logLevel: LogLevel) => void;
 };

package/dist/audio-extraction/audio-manager.js

@@ -1,3 +1,4 @@
+import { Internals } from 'remotion';
 import { getMaxVideoCacheSize, getTotalCacheStats } from '../caches';
 import { makeAudioIterator } from './audio-iterator';
 export const makeAudioManager = () => {
@@ -26,17 +27,30 @@ export const makeAudioManager = () => {
         }
         return mostInThePastIterator;
     };
-    const deleteOldestIterator = async () => {
+    const deleteOldestIterator = () => {
         const iterator = getIteratorMostInThePast();
         if (iterator) {
-            await iterator.prepareForDeletion();
+            iterator.prepareForDeletion();
             iterators.splice(iterators.indexOf(iterator), 1);
         }
     };
+    const deleteDuplicateIterators = (logLevel) => {
+        const seenKeys = new Set();
+        for (let i = 0; i < iterators.length; i++) {
+            const iterator = iterators[i];
+            const key = `${iterator.src}-${iterator.getOldestTimestamp()}-${iterator.getNewestTimestamp()}`;
+            if (seenKeys.has(key)) {
+                iterator.prepareForDeletion();
+                iterators.splice(iterators.indexOf(iterator), 1);
+                Internals.Log.verbose({ logLevel, tag: '@remotion/media' }, `Deleted duplicate iterator for ${iterator.src}`);
+            }
+            seenKeys.add(key);
+        }
+    };
     const getIterator = async ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }) => {
         const maxCacheSize = getMaxVideoCacheSize(logLevel);
         while ((await getTotalCacheStats()).totalSize > maxCacheSize) {
-            await deleteOldestIterator();
+            deleteOldestIterator();
         }
         for (const iterator of iterators) {
             if (iterator.src === src &&
@@ -45,13 +59,15 @@ export const makeAudioManager = () => {
                 return iterator;
             }
         }
-        for (const iterator of iterators) {
-            // delete iterator with same starting timestamp
+        for (let i = 0; i < iterators.length; i++) {
+            const iterator = iterators[i];
+            // delete iterator with same starting timestamp as requested
             if (iterator.src === src && iterator.startTimestamp === timeInSeconds) {
-                await iterator.prepareForDeletion();
+                iterator.prepareForDeletion();
                 iterators.splice(iterators.indexOf(iterator), 1);
             }
         }
+        deleteDuplicateIterators(logLevel);
         return makeIterator({
             src,
             timeInSeconds,
@@ -76,11 +92,22 @@ export const makeAudioManager = () => {
             iterator.logOpenFrames();
         }
     };
+    let queue = Promise.resolve(undefined);
     return {
-        makeIterator,
-        getIterator,
+        getIterator: ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }) => {
+            queue = queue.then(() => getIterator({
+                src,
+                timeInSeconds,
+                audioSampleSink,
+                isMatroska,
+                actualMatroskaTimestamps,
+                logLevel,
+            }));
+            return queue;
+        },
         getCacheStats,
         getIteratorMostInThePast,
         logOpenFrames,
+        deleteDuplicateIterators,
     };
 };

package/dist/audio-extraction/extract-audio.d.ts

@@ -1,6 +1,7 @@
 import { type LogLevel } from 'remotion';
 import type { PcmS16AudioData } from '../convert-audiodata/convert-audiodata';
-export declare const extractAudio: ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds, logLevel, loop, playbackRate, audioStreamIndex, }: {
+type ExtractAudioReturnType = Awaited<ReturnType<typeof extractAudioInternal>>;
+type ExtractAudioParams = {
     src: string;
     timeInSeconds: number;
     durationInSeconds: number;
@@ -8,7 +9,13 @@ export declare const extractAudio: ({ src, timeInSeconds: unloopedTimeInSeconds,
     loop: boolean;
     playbackRate: number;
     audioStreamIndex: number;
-}) => Promise<{
+    trimBefore: number | undefined;
+    trimAfter: number | undefined;
+    fps: number;
+};
+declare const extractAudioInternal: ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds: durationNotYetApplyingPlaybackRate, logLevel, loop, playbackRate, audioStreamIndex, trimBefore, trimAfter, fps, }: ExtractAudioParams) => Promise<{
     data: PcmS16AudioData | null;
     durationInSeconds: number | null;
 } | "cannot-decode" | "unknown-container-format">;
+export declare const extractAudio: (params: ExtractAudioParams) => Promise<ExtractAudioReturnType>;
+export {};

package/dist/audio-extraction/extract-audio.js

@@ -1,13 +1,13 @@
 import { audioManager } from '../caches';
 import { combineAudioDataAndClosePrevious } from '../convert-audiodata/combine-audiodata';
 import { convertAudioData } from '../convert-audiodata/convert-audiodata';
-import { TARGET_NUMBER_OF_CHANNELS, TARGET_SAMPLE_RATE, } from '../convert-audiodata/resample-audiodata';
-import { getSinkWeak } from '../get-sink-weak';
-export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds, logLevel, loop, playbackRate, audioStreamIndex, }) => {
-    const { getAudio, actualMatroskaTimestamps, isMatroska, getDuration } = await getSinkWeak(src, logLevel);
-    let duration = null;
+import { getSink } from '../get-sink';
+import { getTimeInSeconds } from '../get-time-in-seconds';
+const extractAudioInternal = async ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds: durationNotYetApplyingPlaybackRate, logLevel, loop, playbackRate, audioStreamIndex, trimBefore, trimAfter, fps, }) => {
+    const { getAudio, actualMatroskaTimestamps, isMatroska, getDuration } = await getSink(src, logLevel);
+    let mediaDurationInSeconds = null;
     if (loop) {
-        duration = await getDuration();
+        mediaDurationInSeconds = await getDuration();
     }
     const audio = await getAudio(audioStreamIndex);
     if (audio === 'no-audio-track') {
@@ -19,9 +19,20 @@ export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds,
     if (audio === 'unknown-container-format') {
         return 'unknown-container-format';
     }
-    const timeInSeconds = loop
-        ? unloopedTimeInSeconds % duration
-        : unloopedTimeInSeconds;
+    const timeInSeconds = getTimeInSeconds({
+        loop,
+        mediaDurationInSeconds,
+        unloopedTimeInSeconds,
+        src,
+        trimAfter,
+        playbackRate,
+        trimBefore,
+        fps,
+        ifNoMediaDuration: 'fail',
+    });
+    if (timeInSeconds === null) {
+        return { data: null, durationInSeconds: mediaDurationInSeconds };
+    }
     const sampleIterator = await audioManager.getIterator({
         src,
         timeInSeconds,
@@ -30,6 +41,7 @@ export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds,
         actualMatroskaTimestamps,
         logLevel,
     });
+    const durationInSeconds = durationNotYetApplyingPlaybackRate * playbackRate;
     const samples = await sampleIterator.getSamples(timeInSeconds, durationInSeconds);
     audioManager.logOpenFrames();
     const audioDataArray = [];
@@ -51,14 +63,13 @@ export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds,
         // amount of samples to shave from start and end
         let trimStartInSeconds = 0;
         let trimEndInSeconds = 0;
-        // TODO: Apply tone frequency
         if (isFirstSample) {
             trimStartInSeconds = timeInSeconds - sample.timestamp;
             if (trimStartInSeconds < 0 && trimStartInSeconds > -1e-10) {
                 trimStartInSeconds = 0;
             }
             if (trimStartInSeconds < 0) {
-                throw new Error(`trimStartInSeconds is negative: ${trimStartInSeconds}`);
+                throw new Error(`trimStartInSeconds is negative: ${trimStartInSeconds}. ${JSON.stringify({ timeInSeconds, ts: sample.timestamp, d: sample.duration, isFirstSample, isLastSample, durationInSeconds, i, st: samples.map((s) => s.timestamp) })}`);
             }
         }
         if (isLastSample) {
@@ -70,10 +81,8 @@ export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds,
         }
         const audioData = convertAudioData({
             audioData: audioDataRaw,
-            newSampleRate: TARGET_SAMPLE_RATE,
             trimStartInSeconds,
             trimEndInSeconds,
-            targetNumberOfChannels: TARGET_NUMBER_OF_CHANNELS,
             playbackRate,
         });
         audioDataRaw.close();
@@ -83,8 +92,13 @@ export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds,
         audioDataArray.push(audioData);
     }
     if (audioDataArray.length === 0) {
-        return { data: null, durationInSeconds: duration };
+        return { data: null, durationInSeconds: mediaDurationInSeconds };
     }
     const combined = combineAudioDataAndClosePrevious(audioDataArray);
-    return { data: combined, durationInSeconds: duration };
+    return { data: combined, durationInSeconds: mediaDurationInSeconds };
+};
+let queue = Promise.resolve(undefined);
+export const extractAudio = (params) => {
+    queue = queue.then(() => extractAudioInternal(params));
+    return queue;
 };

package/dist/caches.d.ts

@@ -2,45 +2,19 @@ import { type LogLevel } from 'remotion';
 export declare const SAFE_BACK_WINDOW_IN_SECONDS = 1;
 export declare const keyframeManager: {
     requestKeyframeBank: ({ packetSink, timestamp, videoSampleSink, src, logLevel, }: {
-        timestamp: number;
         packetSink: import("mediabunny").EncodedPacketSink;
+        timestamp: number;
         videoSampleSink: import("mediabunny").VideoSampleSink;
         src: string;
         logLevel: LogLevel;
-    }) => Promise<import("./video-extraction/keyframe-bank").KeyframeBank>;
-    addKeyframeBank: ({ src, bank, startTimestampInSeconds, }: {
-        src: string;
-        bank: Promise<import("./video-extraction/keyframe-bank").KeyframeBank>;
-        startTimestampInSeconds: number;
-    }) => void;
+    }) => Promise<import("./video-extraction/keyframe-bank").KeyframeBank | null>;
     getCacheStats: () => Promise<{
         count: number;
         totalSize: number;
     }>;
-    clearAll: () => Promise<void>;
+    clearAll: (logLevel: LogLevel) => Promise<void>;
 };
 export declare const audioManager: {
-    makeIterator: ({ timeInSeconds, src, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }: {
-        timeInSeconds: number;
-        src: string;
-        audioSampleSink: import("mediabunny").AudioSampleSink;
-        isMatroska: boolean;
-        actualMatroskaTimestamps: import("./video-extraction/remember-actual-matroska-timestamps").RememberActualMatroskaTimestamps;
-        logLevel: LogLevel;
-    }) => {
-        src: string;
-        getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
-        waitForCompletion: () => Promise<boolean>;
-        canSatisfyRequestedTime: (timestamp: number) => boolean;
-        logOpenFrames: () => void;
-        getCacheStats: () => {
-            count: number;
-            size: number;
-        };
-        getLastUsed: () => number;
-        prepareForDeletion: () => Promise<void>;
-        startTimestamp: number;
-    };
     getIterator: ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, logLevel, }: {
         src: string;
         timeInSeconds: number;
@@ -48,20 +22,7 @@ export declare const audioManager: {
         isMatroska: boolean;
         actualMatroskaTimestamps: import("./video-extraction/remember-actual-matroska-timestamps").RememberActualMatroskaTimestamps;
         logLevel: LogLevel;
-    }) => Promise<{
-        src: string;
-        getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
-        waitForCompletion: () => Promise<boolean>;
-        canSatisfyRequestedTime: (timestamp: number) => boolean;
-        logOpenFrames: () => void;
-        getCacheStats: () => {
-            count: number;
-            size: number;
-        };
-        getLastUsed: () => number;
-        prepareForDeletion: () => Promise<void>;
-        startTimestamp: number;
-    }>;
+    }) => Promise<import("./audio-extraction/audio-iterator").AudioSampleIterator>;
     getCacheStats: () => {
         count: number;
         totalSize: number;
@@ -77,10 +38,14 @@ export declare const audioManager: {
             size: number;
         };
         getLastUsed: () => number;
-        prepareForDeletion: () => Promise<void>;
+        prepareForDeletion: () => void;
         startTimestamp: number;
+        clearBeforeThreshold: (threshold: number) => void;
+        getOldestTimestamp: () => number;
+        getNewestTimestamp: () => number | null;
     } | null;
     logOpenFrames: () => void;
+    deleteDuplicateIterators: (logLevel: LogLevel) => void;
 };
 export declare const getTotalCacheStats: () => Promise<{
     count: number;

package/dist/convert-audiodata/combine-audiodata.js

@@ -1,30 +1,11 @@
+import { TARGET_NUMBER_OF_CHANNELS } from './resample-audiodata';
 export const combineAudioDataAndClosePrevious = (audioDataArray) => {
     let numberOfFrames = 0;
-    let numberOfChannels = null;
-    let sampleRate = null;
     const { timestamp } = audioDataArray[0];
     for (const audioData of audioDataArray) {
         numberOfFrames += audioData.numberOfFrames;
-        if (!numberOfChannels) {
-            numberOfChannels = audioData.numberOfChannels;
-        }
-        else if (numberOfChannels !== audioData.numberOfChannels) {
-            throw new Error('Number of channels do not match');
-        }
-        if (!sampleRate) {
-            sampleRate = audioData.sampleRate;
-        }
-        else if (sampleRate !== audioData.sampleRate) {
-            throw new Error('Sample rates do not match');
-        }
     }
-    if (!numberOfChannels) {
-        throw new Error('Number of channels is not set');
-    }
-    if (!sampleRate) {
-        throw new Error('Sample rate is not set');
-    }
-    const arr = new Int16Array(numberOfFrames * numberOfChannels);
+    const arr = new Int16Array(numberOfFrames * TARGET_NUMBER_OF_CHANNELS);
     let offset = 0;
     for (const audioData of audioDataArray) {
         arr.set(audioData.data, offset);
@@ -32,9 +13,7 @@ export const combineAudioDataAndClosePrevious = (audioDataArray) => {
     }
     return {
         data: arr,
-        numberOfChannels,
         numberOfFrames,
-        sampleRate,
         timestamp,
     };
 };

package/dist/convert-audiodata/convert-audiodata.d.ts

@@ -1,16 +1,12 @@
 export type ConvertAudioDataOptions = {
     audioData: AudioData;
-    newSampleRate: number;
     trimStartInSeconds: number;
     trimEndInSeconds: number;
-    targetNumberOfChannels: number;
     playbackRate: number;
 };
 export type PcmS16AudioData = {
     data: Int16Array;
-    sampleRate: number;
-    numberOfChannels: number;
     numberOfFrames: number;
     timestamp: number;
 };
-export declare const convertAudioData: ({ audioData, newSampleRate, trimStartInSeconds, trimEndInSeconds, targetNumberOfChannels, playbackRate, }: ConvertAudioDataOptions) => PcmS16AudioData;
+export declare const convertAudioData: ({ audioData, trimStartInSeconds, trimEndInSeconds, playbackRate, }: ConvertAudioDataOptions) => PcmS16AudioData;

package/dist/convert-audiodata/convert-audiodata.js

@@ -1,25 +1,22 @@
-import { resampleAudioData } from './resample-audiodata';
+import { resampleAudioData, TARGET_NUMBER_OF_CHANNELS, TARGET_SAMPLE_RATE, } from './resample-audiodata';
 const FORMAT = 's16';
-const roundButRoundDownZeroPointFive = (value) => {
-    if (value % 1 <= 0.5) {
-        return Math.floor(value);
-    }
-    return Math.ceil(value);
-};
-export const convertAudioData = ({ audioData, newSampleRate, trimStartInSeconds, trimEndInSeconds, targetNumberOfChannels, playbackRate, }) => {
+export const convertAudioData = ({ audioData, trimStartInSeconds, trimEndInSeconds, playbackRate, }) => {
     const { numberOfChannels: srcNumberOfChannels, sampleRate: currentSampleRate, numberOfFrames, } = audioData;
-    const ratio = currentSampleRate / newSampleRate;
-    const frameOffset = roundButRoundDownZeroPointFive(trimStartInSeconds * audioData.sampleRate);
+    const ratio = currentSampleRate / TARGET_SAMPLE_RATE;
+    // Always rounding down start timestamps and rounding up end durations
+    // to ensure there are no gaps when the samples don't align
+    // In @remotion/renderer inline audio mixing, we also round down the sample start
+    // timestamp and round up the end timestamp
+    // This might lead to overlapping, hopefully aligning perfectly!
+    // Test case: https://github.com/remotion-dev/remotion/issues/5758
+    const frameOffset = Math.floor(trimStartInSeconds * audioData.sampleRate);
     const unroundedFrameCount = numberOfFrames -
         (trimEndInSeconds + trimStartInSeconds) * audioData.sampleRate;
-    const frameCount = Math.round(unroundedFrameCount);
-    const newNumberOfFrames = Math.round(unroundedFrameCount / ratio / playbackRate);
+    const frameCount = Math.ceil(unroundedFrameCount);
+    const newNumberOfFrames = Math.ceil(unroundedFrameCount / ratio / playbackRate);
     if (newNumberOfFrames === 0) {
         throw new Error('Cannot resample - the given sample rate would result in less than 1 sample');
     }
-    if (newSampleRate < 3000 || newSampleRate > 768000) {
-        throw new Error('newSampleRate must be between 3000 and 768000');
-    }
     const srcChannels = new Int16Array(srcNumberOfChannels * frameCount);
     audioData.copyTo(srcChannels, {
         planeIndex: 0,
@@ -27,17 +24,15 @@ export const convertAudioData = ({ audioData, newSampleRate, trimStartInSeconds,
         frameOffset,
         frameCount,
     });
-    const data = new Int16Array(newNumberOfFrames * targetNumberOfChannels);
+    const data = new Int16Array(newNumberOfFrames * TARGET_NUMBER_OF_CHANNELS);
     const chunkSize = frameCount / newNumberOfFrames;
     if (newNumberOfFrames === frameCount &&
-        targetNumberOfChannels === srcNumberOfChannels &&
+        TARGET_NUMBER_OF_CHANNELS === srcNumberOfChannels &&
         playbackRate === 1) {
         return {
             data: srcChannels,
-            numberOfChannels: targetNumberOfChannels,
             numberOfFrames: newNumberOfFrames,
-            sampleRate: newSampleRate,
-            timestamp: audioData.timestamp + trimStartInSeconds * 1000000,
+            timestamp: audioData.timestamp + (frameOffset / audioData.sampleRate) * 1000000,
         };
     }
     resampleAudioData({
@@ -49,11 +44,8 @@ export const convertAudioData = ({ audioData, newSampleRate, trimStartInSeconds,
     });
     const newAudioData = {
         data,
-        format: FORMAT,
-        numberOfChannels: targetNumberOfChannels,
         numberOfFrames: newNumberOfFrames,
-        sampleRate: newSampleRate,
-        timestamp: audioData.timestamp + trimStartInSeconds * 1000000,
+        timestamp: audioData.timestamp + (frameOffset / audioData.sampleRate) * 1000000,
     };
     return newAudioData;
 };