@remotion/media-utils 4.0.381 → 4.0.383

package/dist/get-partial-audio-data.d.ts

@@ -0,0 +1,10 @@
+import type { InputAudioTrack } from 'mediabunny';
+export type GetPartialAudioDataProps = {
+    track: InputAudioTrack;
+    fromSeconds: number;
+    toSeconds: number;
+    channelIndex: number;
+    signal: AbortSignal;
+    isMatroska?: boolean;
+};
+export declare const getPartialAudioData: ({ track, fromSeconds, toSeconds, channelIndex, signal, isMatroska, }: GetPartialAudioDataProps) => Promise<Float32Array>;

package/dist/get-partial-audio-data.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPartialAudioData = void 0;
+const mediabunny_1 = require("mediabunny");
+// Audio frames might have dependencies on previous and next frames so we need to decode a bit more and then discard it.
+// The worst case seems to be FLAC files with a 65'535 sample window, which would be 1486.0ms at 44.1Khz.
+// So let's set a threshold of 1.5 seconds.
+const EXTRA_THRESHOLD_IN_SECONDS = 1.5;
+const getPartialAudioData = async ({ track, fromSeconds, toSeconds, channelIndex, signal, isMatroska = false, }) => {
+    if (signal.aborted) {
+        throw new Error('Operation was aborted');
+    }
+    const audioSamples = [];
+    // matroska must be decoded from the start due to limitation
+    // https://www.remotion.dev/docs/media/support#matroska-limitation
+    // Also request extra data beforehand to handle audio frame dependencies
+    const actualFromSeconds = isMatroska
+        ? 0
+        : Math.max(0, fromSeconds - EXTRA_THRESHOLD_IN_SECONDS);
+    // mediabunny docs: constructing the sink is virtually free and does not perform any media data reads.
+    const sink = new mediabunny_1.AudioBufferSink(track);
+    for await (const { buffer, timestamp, duration } of sink.buffers(actualFromSeconds, toSeconds)) {
+        if (signal.aborted) {
+            break;
+        }
+        const channelData = buffer.getChannelData(channelIndex);
+        const bufferStartSeconds = timestamp;
+        const bufferEndSeconds = timestamp + duration;
+        const overlapStartSecond = Math.max(bufferStartSeconds, fromSeconds);
+        const overlapEndSecond = Math.min(bufferEndSeconds, toSeconds);
+        if (overlapStartSecond >= overlapEndSecond) {
+            continue;
+        }
+        const startSampleInBuffer = Math.floor((overlapStartSecond - bufferStartSeconds) * buffer.sampleRate);
+        const endSampleInBuffer = Math.ceil((overlapEndSecond - bufferStartSeconds) * buffer.sampleRate);
+        const trimmedData = channelData.slice(startSampleInBuffer, endSampleInBuffer);
+        audioSamples.push(trimmedData);
+    }
+    const totalSamples = audioSamples.reduce((sum, sample) => sum + sample.length, 0);
+    const result = new Float32Array(totalSamples);
+    let offset = 0;
+    for (const audioSample of audioSamples) {
+        result.set(audioSample, offset);
+        offset += audioSample.length;
+    }
+    return result;
+};
+exports.getPartialAudioData = getPartialAudioData;

package/dist/index.d.ts

@@ -3,10 +3,8 @@ export { createSmoothSvgPath } from './create-smooth-svg-path';
 export { getAudioData } from './get-audio-data';
 export { getAudioDuration, getAudioDurationInSeconds, } from './get-audio-duration-in-seconds';
 export { getImageDimensions } from './get-image-dimensions';
-export { getPartialWaveData } from './get-partial-wave-data';
 export { getVideoMetadata } from './get-video-metadata';
 export { getWaveformPortion } from './get-waveform-portion';
-export { WaveProbe, probeWaveFile } from './probe-wave-file';
 export * from './types';
 export type { AudioData, MediaUtilsAudioData, VideoMetadata as VideoData, } from './types';
 export { useAudioData } from './use-audio-data';

package/dist/index.js

@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.visualizeAudioWaveform = exports.visualizeAudio = exports.useWindowedAudioData = exports.useAudioData = exports.probeWaveFile = exports.getWaveformPortion = exports.getVideoMetadata = exports.getPartialWaveData = exports.getImageDimensions = exports.getAudioDurationInSeconds = exports.getAudioDuration = exports.getAudioData = exports.createSmoothSvgPath = exports.audioBufferToDataUrl = void 0;
+exports.visualizeAudioWaveform = exports.visualizeAudio = exports.useWindowedAudioData = exports.useAudioData = exports.getWaveformPortion = exports.getVideoMetadata = exports.getImageDimensions = exports.getAudioDurationInSeconds = exports.getAudioDuration = exports.getAudioData = exports.createSmoothSvgPath = exports.audioBufferToDataUrl = void 0;
 var audio_url_helpers_1 = require("./audio-buffer/audio-url-helpers");
 Object.defineProperty(exports, "audioBufferToDataUrl", { enumerable: true, get: function () { return audio_url_helpers_1.audioBufferToDataUrl; } });
 var create_smooth_svg_path_1 = require("./create-smooth-svg-path");
@@ -26,14 +26,10 @@ Object.defineProperty(exports, "getAudioDuration", { enumerable: true, get: func
 Object.defineProperty(exports, "getAudioDurationInSeconds", { enumerable: true, get: function () { return get_audio_duration_in_seconds_1.getAudioDurationInSeconds; } });
 var get_image_dimensions_1 = require("./get-image-dimensions");
 Object.defineProperty(exports, "getImageDimensions", { enumerable: true, get: function () { return get_image_dimensions_1.getImageDimensions; } });
-var get_partial_wave_data_1 = require("./get-partial-wave-data");
-Object.defineProperty(exports, "getPartialWaveData", { enumerable: true, get: function () { return get_partial_wave_data_1.getPartialWaveData; } });
 var get_video_metadata_1 = require("./get-video-metadata");
 Object.defineProperty(exports, "getVideoMetadata", { enumerable: true, get: function () { return get_video_metadata_1.getVideoMetadata; } });
 var get_waveform_portion_1 = require("./get-waveform-portion");
 Object.defineProperty(exports, "getWaveformPortion", { enumerable: true, get: function () { return get_waveform_portion_1.getWaveformPortion; } });
-var probe_wave_file_1 = require("./probe-wave-file");
-Object.defineProperty(exports, "probeWaveFile", { enumerable: true, get: function () { return probe_wave_file_1.probeWaveFile; } });
 __exportStar(require("./types"), exports);
 var use_audio_data_1 = require("./use-audio-data");
 Object.defineProperty(exports, "useAudioData", { enumerable: true, get: function () { return use_audio_data_1.useAudioData; } });

package/dist/use-windowed-audio-data.js

@@ -1,15 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.useWindowedAudioData = void 0;
+const mediabunny_1 = require("mediabunny");
 const react_1 = require("react");
 const remotion_1 = require("remotion");
 const combine_float32_arrays_1 = require("./combine-float32-arrays");
-const get_partial_wave_data_1 = require("./get-partial-wave-data");
+const get_partial_audio_data_1 = require("./get-partial-audio-data");
 const is_remote_asset_1 = require("./is-remote-asset");
-const probe_wave_file_1 = require("./probe-wave-file");
+const warnedMatroska = {};
 const useWindowedAudioData = ({ src, frame, fps, windowInSeconds, channelIndex = 0, }) => {
     const isMounted = (0, react_1.useRef)(true);
-    const [waveProbe, setWaveProbe] = (0, react_1.useState)(null);
+    const [audioUtils, setAudioUtils] = (0, react_1.useState)(null);
     const [waveFormMap, setWaveformMap] = (0, react_1.useState)({});
     const requests = (0, react_1.useRef)({});
     const [initialWindowInSeconds] = (0, react_1.useState)(windowInSeconds);
@@ -20,8 +21,18 @@ const useWindowedAudioData = ({ src, frame, fps, windowInSeconds, channelIndex =
         isMounted.current = true;
         return () => {
             isMounted.current = false;
+            Object.values(requests.current).forEach((controller) => {
+                if (controller) {
+                    controller.abort();
+                }
+            });
+            requests.current = {};
+            setWaveformMap({});
+            if (audioUtils) {
+                audioUtils.input.dispose();
+            }
         };
-    }, []);
+    }, [audioUtils]);
     const { delayRender, continueRender } = (0, remotion_1.useDelayRender)();
     const fetchMetadata = (0, react_1.useCallback)(async (signal) => {
         const handle = delayRender(`Waiting for audio metadata with src="${src}" to be loaded`);
@@ -29,20 +40,53 @@ const useWindowedAudioData = ({ src, frame, fps, windowInSeconds, channelIndex =
             continueRender(handle);
         };
         signal.addEventListener('abort', cont, { once: true });
+        const input = new mediabunny_1.Input({
+            formats: mediabunny_1.ALL_FORMATS,
+            source: new mediabunny_1.UrlSource(src),
+        });
+        const onAbort = () => {
+            input.dispose();
+        };
+        signal.addEventListener('abort', onAbort, { once: true });
         try {
-            const data = await (0, probe_wave_file_1.probeWaveFile)(src);
+            const durationInSeconds = await input.computeDuration();
+            const audioTrack = await input.getPrimaryAudioTrack();
+            if (!audioTrack) {
+                throw new Error('No audio track found');
+            }
+            const canDecode = await audioTrack.canDecode();
+            if (!canDecode) {
+                throw new Error('Audio track cannot be decoded');
+            }
+            if (channelIndex >= audioTrack.numberOfChannels || channelIndex < 0) {
+                throw new Error(`Invalid channel index ${channelIndex} for audio with ${audioTrack.numberOfChannels} channels`);
+            }
+            const { numberOfChannels, sampleRate } = audioTrack;
+            const format = await input.getFormat();
+            const isMatroska = format === mediabunny_1.MATROSKA || format === mediabunny_1.WEBM;
             if (isMounted.current) {
-                setWaveProbe(data);
+                setAudioUtils({
+                    input,
+                    track: audioTrack,
+                    metadata: {
+                        durationInSeconds,
+                        numberOfChannels,
+                        sampleRate,
+                    },
+                    isMatroska,
+                });
             }
             continueRender(handle);
         }
         catch (err) {
+            input.dispose();
             (0, remotion_1.cancelRender)(err);
         }
         finally {
             signal.removeEventListener('abort', cont);
+            signal.removeEventListener('abort', onAbort);
         }
-    }, [src, delayRender, continueRender]);
+    }, [src, delayRender, continueRender, channelIndex]);
     (0, react_1.useLayoutEffect)(() => {
         const controller = new AbortController();
         fetchMetadata(controller.signal);
@@ -53,13 +97,13 @@ const useWindowedAudioData = ({ src, frame, fps, windowInSeconds, channelIndex =
     const currentTime = frame / fps;
     const currentWindowIndex = Math.floor(currentTime / windowInSeconds);
     const windowsToFetch = (0, react_1.useMemo)(() => {
-        if (!waveProbe) {
+        if (!(audioUtils === null || audioUtils === void 0 ? void 0 : audioUtils.metadata)) {
             return [];
         }
         const maxWindowIndex = Math.floor(
         // If an audio is exactly divisible by windowInSeconds, we need to
         // subtract 0.000000000001 to avoid fetching an extra window.
-        waveProbe.durationInSeconds / windowInSeconds - 0.000000000001);
+        audioUtils.metadata.durationInSeconds / windowInSeconds - 0.000000000001);
         // needs to be in order because we rely on the concatenation below
         return [
             currentWindowIndex === 0 ? null : currentWindowIndex - 1,
@@ -68,43 +112,74 @@ const useWindowedAudioData = ({ src, frame, fps, windowInSeconds, channelIndex =
         ]
             .filter((i) => i !== null)
             .filter((i) => i >= 0);
-    }, [currentWindowIndex, waveProbe, windowInSeconds]);
+    }, [currentWindowIndex, audioUtils === null || audioUtils === void 0 ? void 0 : audioUtils.metadata, windowInSeconds]);
     const fetchAndSetWaveformData = (0, react_1.useCallback)(async (windowIndex) => {
-        if (!waveProbe) {
-            throw new Error('Wave probe is not loaded yet');
+        if (!(audioUtils === null || audioUtils === void 0 ? void 0 : audioUtils.metadata) || !audioUtils) {
+            throw new Error('MediaBunny context is not loaded yet');
+        }
+        // Cancel any existing request for this window, we don't want to over-fetch
+        const existingController = requests.current[windowIndex];
+        if (existingController) {
+            existingController.abort();
         }
         const controller = new AbortController();
         requests.current[windowIndex] = controller;
-        const partialWaveData = await (0, get_partial_wave_data_1.getPartialWaveData)({
-            bitsPerSample: waveProbe.bitsPerSample,
-            blockAlign: waveProbe.blockAlign,
-            channelIndex,
-            dataOffset: waveProbe.dataOffset,
-            fileSize: waveProbe.fileSize,
-            fromSeconds: windowIndex * windowInSeconds,
-            sampleRate: waveProbe.sampleRate,
-            src,
-            toSeconds: (windowIndex + 1) * windowInSeconds,
-            signal: controller.signal,
-        });
-        requests.current[windowIndex] = null;
-        setWaveformMap((prev) => {
-            const entries = Object.keys(prev);
-            const windowsToClear = entries.filter((entry) => !windowsToFetch.includes(Number(entry)));
-            return {
-                ...prev,
-                // Delete windows that are not needed anymore
-                ...windowsToClear.reduce((acc, key) => {
-                    acc[key] = null;
-                    return acc;
-                }, {}),
-                // Add the new window
-                [windowIndex]: partialWaveData,
-            };
-        });
-    }, [channelIndex, src, waveProbe, windowInSeconds, windowsToFetch]);
+        if (controller.signal.aborted) {
+            return;
+        }
+        const fromSeconds = windowIndex * windowInSeconds;
+        const toSeconds = (windowIndex + 1) * windowInSeconds;
+        // if both fromSeconds and toSeconds are outside of the audio duration, skip fetching
+        if (fromSeconds >= audioUtils.metadata.durationInSeconds ||
+            toSeconds <= 0) {
+            return;
+        }
+        try {
+            const { isMatroska } = audioUtils;
+            if (isMatroska && !warnedMatroska[src]) {
+                warnedMatroska[src] = true;
+                remotion_1.Internals.Log.warn({ logLevel: 'info', tag: '@remotion/media-utils' }, `[useWindowedAudioData] Matroska/WebM file detected at "${src}".\n\nDue to format limitation, audio decoding must start from the beginning of the file, which may lead to increased memory usage and slower performance for large files. Consider converting the audio to a more suitable format like MP3 or AAC for better performance.`);
+            }
+            const partialWaveData = await (0, get_partial_audio_data_1.getPartialAudioData)({
+                track: audioUtils.track,
+                fromSeconds,
+                toSeconds,
+                channelIndex,
+                signal: controller.signal,
+                isMatroska,
+            });
+            if (!controller.signal.aborted) {
+                setWaveformMap((prev) => {
+                    const entries = Object.keys(prev);
+                    const windowsToClear = entries.filter((entry) => !windowsToFetch.includes(Number(entry)));
+                    return {
+                        ...prev,
+                        ...windowsToClear.reduce((acc, key) => {
+                            acc[key] = null;
+                            return acc;
+                        }, {}),
+                        [windowIndex]: partialWaveData,
+                    };
+                });
+            }
+        }
+        catch (err) {
+            if (controller.signal.aborted) {
+                return;
+            }
+            if (err instanceof mediabunny_1.InputDisposedError) {
+                return;
+            }
+            throw err;
+        }
+        finally {
+            if (requests.current[windowIndex] === controller) {
+                requests.current[windowIndex] = null;
+            }
+        }
+    }, [channelIndex, audioUtils, windowInSeconds, windowsToFetch, src]);
     (0, react_1.useEffect)(() => {
-        if (!waveProbe) {
+        if (!(audioUtils === null || audioUtils === void 0 ? void 0 : audioUtils.metadata)) {
             return;
         }
         const windowsToClear = Object.keys(requests.current).filter((entry) => !windowsToFetch.includes(Number(entry)));
@@ -115,7 +190,12 @@ const useWindowedAudioData = ({ src, frame, fps, windowInSeconds, channelIndex =
                 requests.current[windowIndex] = null;
             }
         }
-        Promise.all(windowsToFetch.map((windowIndex) => {
+        // Only fetch windows that don't already exist
+        const windowsToActuallyFetch = windowsToFetch.filter((windowIndex) => !waveFormMap[windowIndex]);
+        if (windowsToActuallyFetch.length === 0) {
+            return;
+        }
+        Promise.all(windowsToActuallyFetch.map((windowIndex) => {
             return fetchAndSetWaveformData(windowIndex);
         })).catch((err) => {
             var _a, _b, _c, _d, _e;
@@ -131,25 +211,29 @@ const useWindowedAudioData = ({ src, frame, fps, windowInSeconds, channelIndex =
             }
             (0, remotion_1.cancelRender)(err);
         });
-    }, [fetchAndSetWaveformData, waveProbe, windowsToFetch]);
+    }, [fetchAndSetWaveformData, audioUtils, windowsToFetch, waveFormMap]);
+    // Calculate available windows for reuse
+    const availableWindows = (0, react_1.useMemo)(() => {
+        return windowsToFetch.filter((i) => waveFormMap[i]);
+    }, [windowsToFetch, waveFormMap]);
     const currentAudioData = (0, react_1.useMemo)(() => {
-        if (!waveProbe) {
+        if (!(audioUtils === null || audioUtils === void 0 ? void 0 : audioUtils.metadata)) {
             return null;
         }
-        if (windowsToFetch.some((i) => !waveFormMap[i])) {
+        if (availableWindows.length === 0) {
             return null;
         }
-        const windows = windowsToFetch.map((i) => waveFormMap[i]);
+        const windows = availableWindows.map((i) => waveFormMap[i]);
         const data = (0, combine_float32_arrays_1.combineFloat32Arrays)(windows);
         return {
             channelWaveforms: [data],
-            durationInSeconds: waveProbe.durationInSeconds,
+            durationInSeconds: audioUtils.metadata.durationInSeconds,
             isRemote: (0, is_remote_asset_1.isRemoteAsset)(src),
             numberOfChannels: 1,
-            resultId: String(Math.random()),
-            sampleRate: waveProbe.sampleRate,
+            resultId: `${src}-windows-${availableWindows.join(',')}`,
+            sampleRate: audioUtils.metadata.sampleRate,
         };
-    }, [src, waveFormMap, waveProbe, windowsToFetch]);
+    }, [src, waveFormMap, audioUtils === null || audioUtils === void 0 ? void 0 : audioUtils.metadata, availableWindows]);
     (0, react_1.useLayoutEffect)(() => {
         if (currentAudioData) {
             return;
@@ -159,9 +243,13 @@ const useWindowedAudioData = ({ src, frame, fps, windowInSeconds, channelIndex =
             continueRender(handle);
         };
     }, [currentAudioData, src, delayRender, continueRender]);
+    const isBeyondAudioDuration = audioUtils
+        ? currentTime >= audioUtils.metadata.durationInSeconds
+        : false;
+    const audioData = isBeyondAudioDuration ? null : currentAudioData;
     return {
-        audioData: currentAudioData,
-        dataOffsetInSeconds: windowsToFetch[0] * windowInSeconds,
+        audioData,
+        dataOffsetInSeconds: availableWindows.length > 0 ? availableWindows[0] * windowInSeconds : 0,
     };
 };
 exports.useWindowedAudioData = useWindowedAudioData;

package/package.json

@@ -3,7 +3,7 @@
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/media-utils"
 	},
 	"name": "@remotion/media-utils",
-	"version": "4.0.381",
+	"version": "4.0.383",
 	"description": "Utilities for working with media files",
 	"main": "dist/index.js",
 	"sideEffects": false,
@@ -18,14 +18,17 @@
 		"url": "https://github.com/remotion-dev/remotion/issues"
 	},
 	"dependencies": {
-		"remotion": "4.0.381"
+		"@remotion/media-parser": "4.0.383",
+		"@remotion/webcodecs": "4.0.383",
+		"remotion": "4.0.383",
+		"mediabunny": "1.25.3"
 	},
 	"peerDependencies": {
 		"react": ">=16.8.0",
 		"react-dom": ">=16.8.0"
 	},
 	"devDependencies": {
-		"@remotion/eslint-config-internal": "4.0.381",
+		"@remotion/eslint-config-internal": "4.0.383",
 		"eslint": "9.19.0"
 	},
 	"keywords": [