@remotion/media 4.0.354 → 4.0.356

package/dist/video/video-for-preview.js

@@ -1,18 +1,41 @@
 import { jsx as _jsx } from "react/jsx-runtime";
+import { ALL_FORMATS, Input, UrlSource } from 'mediabunny';
 import { useContext, useEffect, useMemo, useRef, useState } from 'react';
-import { Internals, useBufferState, useCurrentFrame } from 'remotion';
+import { Internals, Loop, useBufferState, useCurrentFrame, useVideoConfig, } from 'remotion';
 import { MediaPlayer } from './media-player';
-const { useUnsafeVideoConfig, Timeline, SharedAudioContext } = Internals;
-export const NewVideoForPreview = ({ src, style, playbackRate = 1, logLevel = 'info', className, }) => {
+const { useUnsafeVideoConfig, Timeline, SharedAudioContext, useMediaMutedState, useMediaVolumeState, useFrameForVolumeProp, evaluateVolume, warnAboutTooHighVolume, usePreload, } = Internals;
+const calculateLoopDuration = ({ endAt, mediaDuration, playbackRate, startFrom, }) => {
+    let duration = mediaDuration;
+    if (typeof endAt !== 'undefined') {
+        duration = endAt;
+    }
+    if (typeof startFrom !== 'undefined') {
+        duration -= startFrom;
+    }
+    const actualDuration = duration / playbackRate;
+    return Math.floor(actualDuration);
+};
+const NewVideoForPreview = ({ src, style, playbackRate, logLevel, className, muted, volume, loopVolumeCurveBehavior, onVideoFrame, }) => {
     const canvasRef = useRef(null);
     const videoConfig = useUnsafeVideoConfig();
     const frame = useCurrentFrame();
     const mediaPlayerRef = useRef(null);
     const [mediaPlayerReady, setMediaPlayerReady] = useState(false);
     const [playing] = Timeline.usePlayingState();
+    const timelineContext = useContext(Timeline.TimelineContext);
+    const globalPlaybackRate = timelineContext.playbackRate;
     const sharedAudioContext = useContext(SharedAudioContext);
     const buffer = useBufferState();
     const delayHandleRef = useRef(null);
+    const [mediaMuted] = useMediaMutedState();
+    const [mediaVolume] = useMediaVolumeState();
+    const volumePropFrame = useFrameForVolumeProp(loopVolumeCurveBehavior ?? 'repeat');
+    const userPreferredVolume = evaluateVolume({
+        frame: volumePropFrame,
+        volume,
+        mediaVolume,
+    });
+    warnAboutTooHighVolume(userPreferredVolume);
     if (!videoConfig) {
         throw new Error('No video config found');
     }
@@ -22,6 +45,7 @@ export const NewVideoForPreview = ({ src, style, playbackRate = 1, logLevel = 'i
     const actualFps = videoConfig.fps / playbackRate;
     const currentTime = frame / actualFps;
     const [initialTimestamp] = useState(currentTime);
+    const preloadedSrc = usePreload(src);
     useEffect(() => {
         if (!canvasRef.current)
             return;
@@ -32,7 +56,7 @@ export const NewVideoForPreview = ({ src, style, playbackRate = 1, logLevel = 'i
         try {
             const player = new MediaPlayer({
                 canvas: canvasRef.current,
-                src,
+                src: preloadedSrc,
                 logLevel,
                 sharedAudioContext: sharedAudioContext.audioContext,
             });
@@ -62,13 +86,12 @@ export const NewVideoForPreview = ({ src, style, playbackRate = 1, logLevel = 'i
             }
             setMediaPlayerReady(false);
         };
-    }, [src, logLevel, sharedAudioContext, initialTimestamp]);
+    }, [preloadedSrc, logLevel, sharedAudioContext, initialTimestamp]);
     const classNameValue = useMemo(() => {
         return [Internals.OBJECTFIT_CONTAIN_CLASS_NAME, className]
             .filter(Internals.truthy)
             .join(' ');
     }, [className]);
-    // sync play/pause state with Remotion timeline (like old VideoForPreview video does)
     useEffect(() => {
         const mediaPlayer = mediaPlayerRef.current;
         if (!mediaPlayer)
@@ -82,7 +105,6 @@ export const NewVideoForPreview = ({ src, style, playbackRate = 1, logLevel = 'i
             mediaPlayer.pause();
         }
     }, [playing, logLevel, mediaPlayerReady]);
-    // sync target time with MediaPlayer
     useEffect(() => {
         const mediaPlayer = mediaPlayerRef.current;
         if (!mediaPlayer || !mediaPlayerReady)
@@ -90,24 +112,107 @@ export const NewVideoForPreview = ({ src, style, playbackRate = 1, logLevel = 'i
         mediaPlayer.seekTo(currentTime);
         Internals.Log.trace({ logLevel, tag: '@remotion/media' }, `[NewVideoForPreview] Updating target time to ${currentTime.toFixed(3)}s`);
     }, [currentTime, logLevel, mediaPlayerReady]);
-    // sync MediaPlayer buffering with Remotion buffering
     useEffect(() => {
         const mediaPlayer = mediaPlayerRef.current;
         if (!mediaPlayer || !mediaPlayerReady)
             return;
         mediaPlayer.onBufferingChange((newBufferingState) => {
             if (newBufferingState && !delayHandleRef.current) {
-                // Start blocking Remotion playback
                 delayHandleRef.current = buffer.delayPlayback();
                 Internals.Log.trace({ logLevel, tag: '@remotion/media' }, '[NewVideoForPreview] MediaPlayer buffering - blocking Remotion playback');
             }
             else if (!newBufferingState && delayHandleRef.current) {
-                // Unblock Remotion playback
                 delayHandleRef.current.unblock();
                 delayHandleRef.current = null;
                 Internals.Log.trace({ logLevel, tag: '@remotion/media' }, '[NewVideoForPreview] MediaPlayer unbuffering - unblocking Remotion playback');
             }
         });
     }, [mediaPlayerReady, buffer, logLevel]);
+    const effectiveMuted = muted || mediaMuted || userPreferredVolume <= 0;
+    useEffect(() => {
+        const mediaPlayer = mediaPlayerRef.current;
+        if (!mediaPlayer || !mediaPlayerReady)
+            return;
+        mediaPlayer.setMuted(effectiveMuted);
+    }, [effectiveMuted, mediaPlayerReady]);
+    useEffect(() => {
+        const mediaPlayer = mediaPlayerRef.current;
+        if (!mediaPlayer || !mediaPlayerReady) {
+            return;
+        }
+        mediaPlayer.setVolume(userPreferredVolume);
+    }, [userPreferredVolume, mediaPlayerReady, logLevel]);
+    const effectivePlaybackRate = useMemo(() => playbackRate * globalPlaybackRate, [playbackRate, globalPlaybackRate]);
+    useEffect(() => {
+        const mediaPlayer = mediaPlayerRef.current;
+        if (!mediaPlayer || !mediaPlayerReady) {
+            return;
+        }
+        mediaPlayer.setPlaybackRate(effectivePlaybackRate).catch((error) => {
+            Internals.Log.error({ logLevel, tag: '@remotion/media' }, '[NewVideoForPreview] Failed to set playback rate', error);
+        });
+    }, [effectivePlaybackRate, mediaPlayerReady, logLevel]);
+    useEffect(() => {
+        const mediaPlayer = mediaPlayerRef.current;
+        if (!mediaPlayer || !mediaPlayerReady) {
+            return;
+        }
+        if (onVideoFrame) {
+            mediaPlayer.onVideoFrame(onVideoFrame);
+        }
+    }, [onVideoFrame, mediaPlayerReady]);
     return (_jsx("canvas", { ref: canvasRef, width: videoConfig.width, height: videoConfig.height, style: style, className: classNameValue }));
 };
+const VideoForPreviewWithDuration = ({ className, durationInSeconds, logLevel, loopVolumeCurveBehavior, muted, onVideoFrame, playbackRate, src, style, volume, loop, name, trimAfter, trimBefore, }) => {
+    const { fps } = useVideoConfig();
+    if (loop) {
+        if (!Number.isFinite(durationInSeconds) || durationInSeconds === null) {
+            return (_jsx(VideoForPreviewWithDuration, { loop: false, className: className, durationInSeconds: durationInSeconds, logLevel: logLevel, loopVolumeCurveBehavior: loopVolumeCurveBehavior, muted: muted, onVideoFrame: onVideoFrame, playbackRate: playbackRate, src: src, style: style, volume: volume, name: name, trimAfter: trimAfter, trimBefore: trimBefore }));
+        }
+        const mediaDuration = durationInSeconds * fps;
+        return (_jsx(Loop, { durationInFrames: calculateLoopDuration({
+                endAt: trimAfter,
+                mediaDuration,
+                playbackRate: playbackRate ?? 1,
+                startFrom: trimBefore,
+            }), layout: "none", name: name, children: _jsx(VideoForPreviewWithDuration, { loop: false, className: className, durationInSeconds: durationInSeconds, logLevel: logLevel, loopVolumeCurveBehavior: loopVolumeCurveBehavior, muted: muted, onVideoFrame: onVideoFrame, playbackRate: playbackRate, src: src, style: style, volume: volume, name: name, trimAfter: trimAfter, trimBefore: trimBefore }) }));
+    }
+    return (_jsx(NewVideoForPreview, { src: src, style: style, playbackRate: playbackRate, logLevel: logLevel, muted: muted, volume: volume, loopVolumeCurveBehavior: loopVolumeCurveBehavior, onVideoFrame: onVideoFrame, className: className }));
+};
+export const VideoForPreview = ({ className, loop, src, logLevel, muted, name, volume, loopVolumeCurveBehavior, onVideoFrame, playbackRate, style, }) => {
+    const preloadedSrc = usePreload(src);
+    const [durationInSeconds, setDurationInSeconds] = useState(null);
+    useEffect(() => {
+        if (!loop) {
+            return;
+        }
+        let cancelled = false;
+        const computeDuration = async () => {
+            const urlSource = new UrlSource(preloadedSrc);
+            const input = new Input({
+                source: urlSource,
+                formats: ALL_FORMATS,
+            });
+            try {
+                const duration = await input.computeDuration();
+                if (!cancelled) {
+                    setDurationInSeconds(duration);
+                }
+            }
+            catch (error) {
+                Internals.Log.error({ logLevel, tag: '@remotion/media' }, '[VideoForPreview] Failed to compute duration', error);
+            }
+            finally {
+                input.dispose();
+            }
+        };
+        computeDuration();
+        return () => {
+            cancelled = true;
+        };
+    }, [loop, preloadedSrc, logLevel]);
+    if (loop && durationInSeconds === null) {
+        return null;
+    }
+    return (_jsx(VideoForPreviewWithDuration, { durationInSeconds: durationInSeconds, className: className, logLevel: logLevel, muted: muted, onVideoFrame: onVideoFrame, playbackRate: playbackRate, src: src, style: style, volume: volume, name: name, trimAfter: undefined, trimBefore: undefined, loop: loop, loopVolumeCurveBehavior: loopVolumeCurveBehavior }));
+};

package/dist/video/video-for-rendering.d.ts

@@ -1,3 +1,24 @@
 import React from 'react';
-import type { VideoProps } from './props';
-export declare const VideoForRendering: React.FC<VideoProps>;
+import type { LogLevel, LoopVolumeCurveBehavior, OnVideoFrame, VolumeProp } from 'remotion';
+import type { FallbackOffthreadVideoProps } from './props';
+type InnerVideoProps = {
+    readonly className: string | undefined;
+    readonly loop: boolean;
+    readonly src: string;
+    readonly logLevel: LogLevel;
+    readonly muted: boolean;
+    readonly name: string | undefined;
+    readonly volume: VolumeProp;
+    readonly loopVolumeCurveBehavior: LoopVolumeCurveBehavior;
+    readonly onVideoFrame: OnVideoFrame | undefined;
+    readonly playbackRate: number;
+    readonly style: React.CSSProperties;
+    readonly delayRenderRetries: number | null;
+    readonly delayRenderTimeoutInMilliseconds: number | null;
+    readonly fallbackOffthreadVideoProps: FallbackOffthreadVideoProps;
+    readonly audioStreamIndex: number;
+    readonly disallowFallbackToOffthreadVideo: boolean;
+    readonly stack: string | undefined;
+};
+export declare const VideoForRendering: React.FC<InnerVideoProps>;
+export {};

package/dist/video/video-for-rendering.js

@@ -4,9 +4,7 @@ import { cancelRender, Internals, useCurrentFrame, useDelayRender, useRemotionEn
 import { applyVolume } from '../convert-audiodata/apply-volume';
 import { frameForVolumeProp } from '../looped-frame';
 import { extractFrameViaBroadcastChannel } from '../video-extraction/extract-frame-via-broadcast-channel';
-export const VideoForRendering = ({ volume: volumeProp, playbackRate, src, muted, loopVolumeCurveBehavior, delayRenderRetries, delayRenderTimeoutInMilliseconds, 
-// call when a frame of the video, i.e. frame drawn on canvas
-onVideoFrame, logLevel = window.remotion_logLevel, loop, style, className, }) => {
+export const VideoForRendering = ({ volume: volumeProp, playbackRate, src, muted, loopVolumeCurveBehavior, delayRenderRetries, delayRenderTimeoutInMilliseconds, onVideoFrame, logLevel, loop, style, className, fallbackOffthreadVideoProps, audioStreamIndex, name, disallowFallbackToOffthreadVideo, stack, }) => {
     if (!src) {
         throw new TypeError('No `src` was passed to <Video>.');
     }
@@ -19,10 +17,14 @@ onVideoFrame, logLevel = window.remotion_logLevel, loop, style, className, }) =>
     const environment = useRemotionEnvironment();
     const { delayRender, continueRender } = useDelayRender();
     const canvasRef = useRef(null);
+    const [replaceWithOffthreadVideo, setReplaceWithOffthreadVideo] = useState(false);
     useLayoutEffect(() => {
         if (!canvasRef.current) {
             return;
         }
+        if (replaceWithOffthreadVideo) {
+            return;
+        }
         const actualFps = playbackRate ? fps / playbackRate : fps;
         const timestamp = frame / actualFps;
         const durationInSeconds = 1 / actualFps;
@@ -49,8 +51,40 @@ onVideoFrame, logLevel = window.remotion_logLevel, loop, style, className, }) =>
             includeVideo: window.remotion_videoEnabled,
             isClientSideRendering: environment.isClientSideRendering,
             loop: loop ?? false,
+            audioStreamIndex: audioStreamIndex ?? 0,
         })
-            .then(({ frame: imageBitmap, audio, durationInSeconds: assetDurationInSeconds, }) => {
+            .then((result) => {
+            if (result === 'unknown-container-format') {
+                if (disallowFallbackToOffthreadVideo) {
+                    cancelRender(new Error(`Unknown container format ${src}, and 'disallowFallbackToOffthreadVideo' was set. Failing the render.`));
+                }
+                if (window.remotion_isMainTab) {
+                    Internals.Log.info({ logLevel, tag: '@remotion/media' }, `Unknown container format for ${src} (Supported formats: https://www.remotion.dev/docs/mediabunny/formats), falling back to <OffthreadVideo>`);
+                }
+                setReplaceWithOffthreadVideo(true);
+                return;
+            }
+            if (result === 'cannot-decode') {
+                if (disallowFallbackToOffthreadVideo) {
+                    cancelRender(new Error(`Cannot decode ${src}, and 'disallowFallbackToOffthreadVideo' was set. Failing the render.`));
+                }
+                if (window.remotion_isMainTab) {
+                    Internals.Log.info({ logLevel, tag: '@remotion/media' }, `Cannot decode ${src}, falling back to <OffthreadVideo>`);
+                }
+                setReplaceWithOffthreadVideo(true);
+                return;
+            }
+            if (result === 'network-error') {
+                if (disallowFallbackToOffthreadVideo) {
+                    cancelRender(new Error(`Cannot decode ${src}, and 'disallowFallbackToOffthreadVideo' was set. Failing the render.`));
+                }
+                if (window.remotion_isMainTab) {
+                    Internals.Log.info({ logLevel, tag: '@remotion/media' }, `Network error fetching ${src}, falling back to <OffthreadVideo>`);
+                }
+                setReplaceWithOffthreadVideo(true);
+                return;
+            }
+            const { frame: imageBitmap, audio, durationInSeconds: assetDurationInSeconds, } = result;
             if (imageBitmap) {
                 onVideoFrame?.(imageBitmap);
                 const context = canvasRef.current?.getContext('2d');
@@ -129,11 +163,20 @@ onVideoFrame, logLevel = window.remotion_logLevel, loop, style, className, }) =>
         startsAt,
         unregisterRenderAsset,
         volumeProp,
+        replaceWithOffthreadVideo,
+        audioStreamIndex,
+        disallowFallbackToOffthreadVideo,
     ]);
     const classNameValue = useMemo(() => {
         return [Internals.OBJECTFIT_CONTAIN_CLASS_NAME, className]
             .filter(Internals.truthy)
             .join(' ');
     }, [className]);
+    if (replaceWithOffthreadVideo) {
+        // TODO: Loop and other props
+        return (_jsx(Internals.InnerOffthreadVideo, { src: src, playbackRate: playbackRate ?? 1, muted: muted ?? false, acceptableTimeShiftInSeconds: fallbackOffthreadVideoProps?.acceptableTimeShiftInSeconds, loopVolumeCurveBehavior: loopVolumeCurveBehavior ?? 'repeat', delayRenderRetries: delayRenderRetries ?? undefined, delayRenderTimeoutInMilliseconds: delayRenderTimeoutInMilliseconds ?? undefined, style: style, allowAmplificationDuringRender: true, transparent: fallbackOffthreadVideoProps?.transparent ?? false, toneMapped: fallbackOffthreadVideoProps?.toneMapped ?? true, audioStreamIndex: audioStreamIndex ?? 0, name: name, className: className, onVideoFrame: onVideoFrame, volume: volumeProp, id: id, onError: fallbackOffthreadVideoProps?.onError, toneFrequency: fallbackOffthreadVideoProps?.toneFrequency ?? 1, 
+            // these shouldn't matter during rendering / should not appear at all
+            showInTimeline: false, crossOrigin: undefined, onAutoPlayError: () => undefined, pauseWhenBuffering: false, trimAfter: undefined, trimBefore: undefined, useWebAudioApi: false, startFrom: undefined, endAt: undefined, stack: stack, _remotionInternalNativeLoopPassed: false }));
+    }
     return _jsx("canvas", { ref: canvasRef, style: style, className: classNameValue });
 };

package/dist/video/video.js

@@ -1,16 +1,12 @@
 import { jsx as _jsx } from "react/jsx-runtime";
-import { useCallback } from 'react';
 import { Internals, Sequence, useRemotionEnvironment } from 'remotion';
+import { VideoForPreview } from './video-for-preview';
 import { VideoForRendering } from './video-for-rendering';
-const { validateMediaTrimProps, resolveTrimProps, validateMediaProps, VideoForPreview, } = Internals;
-export const Video = (props) => {
-    // Should only destruct `trimBefore` and `trimAfter` from props,
-    // rest gets drilled down
-    const { trimBefore, trimAfter, name, pauseWhenBuffering, stack, showInTimeline, ...otherProps } = props;
+const { validateMediaTrimProps, resolveTrimProps, validateMediaProps } = Internals;
+const InnerVideo = ({ src, audioStreamIndex, className, delayRenderRetries, delayRenderTimeoutInMilliseconds, disallowFallbackToOffthreadVideo, fallbackOffthreadVideoProps, logLevel, loop, loopVolumeCurveBehavior, muted, name, onVideoFrame, playbackRate, style, trimAfter, trimBefore, volume, showInTimeline, stack, }) => {
     const environment = useRemotionEnvironment();
-    const onDuration = useCallback(() => undefined, []);
-    if (typeof props.src !== 'string') {
-        throw new TypeError(`The \`<Video>\` tag requires a string for \`src\`, but got ${JSON.stringify(props.src)} instead.`);
+    if (typeof src !== 'string') {
+        throw new TypeError(`The \`<Video>\` tag requires a string for \`src\`, but got ${JSON.stringify(src)} instead.`);
     }
     validateMediaTrimProps({
         startFrom: undefined,
@@ -26,12 +22,15 @@ export const Video = (props) => {
     });
     if (typeof trimBeforeValue !== 'undefined' ||
         typeof trimAfterValue !== 'undefined') {
-        return (_jsx(Sequence, { layout: "none", from: 0 - (trimBeforeValue ?? 0), showInTimeline: false, durationInFrames: trimAfterValue, name: name, children: _jsx(Video, { pauseWhenBuffering: pauseWhenBuffering ?? false, ...otherProps }) }));
+        return (_jsx(Sequence, { layout: "none", from: 0 - (trimBeforeValue ?? 0), showInTimeline: false, durationInFrames: trimAfterValue, name: name, children: _jsx(InnerVideo, { audioStreamIndex: audioStreamIndex, className: className, delayRenderRetries: delayRenderRetries, delayRenderTimeoutInMilliseconds: delayRenderTimeoutInMilliseconds, disallowFallbackToOffthreadVideo: disallowFallbackToOffthreadVideo, name: name, fallbackOffthreadVideoProps: fallbackOffthreadVideoProps, logLevel: logLevel, loop: loop, loopVolumeCurveBehavior: loopVolumeCurveBehavior, muted: muted, onVideoFrame: onVideoFrame, playbackRate: playbackRate, src: src, stack: stack, style: style, volume: volume, trimAfter: undefined, trimBefore: undefined, showInTimeline: showInTimeline }) }));
     }
-    validateMediaProps(props, 'Video');
+    validateMediaProps({ playbackRate, volume }, 'Video');
     if (environment.isRendering) {
-        return _jsx(VideoForRendering, { ...otherProps });
+        return (_jsx(VideoForRendering, { audioStreamIndex: audioStreamIndex ?? 0, className: className, delayRenderRetries: delayRenderRetries ?? null, delayRenderTimeoutInMilliseconds: delayRenderTimeoutInMilliseconds ?? null, disallowFallbackToOffthreadVideo: disallowFallbackToOffthreadVideo ?? false, name: name, fallbackOffthreadVideoProps: fallbackOffthreadVideoProps, logLevel: logLevel, loop: loop, loopVolumeCurveBehavior: loopVolumeCurveBehavior, muted: muted, onVideoFrame: onVideoFrame, playbackRate: playbackRate, src: src, stack: stack, style: style, volume: volume }));
     }
-    const { onVideoFrame, delayRenderRetries, delayRenderTimeoutInMilliseconds, ...propsForPreview } = otherProps;
-    return (_jsx(VideoForPreview, { _remotionInternalStack: stack ?? null, _remotionInternalNativeLoopPassed: false, onDuration: onDuration, onlyWarnForMediaSeekingError: true, pauseWhenBuffering: pauseWhenBuffering ?? false, showInTimeline: showInTimeline ?? true, onVideoFrame: onVideoFrame ?? null, ...propsForPreview }));
+    return (_jsx(VideoForPreview, { className: className, name: name, logLevel: logLevel, loop: loop, loopVolumeCurveBehavior: loopVolumeCurveBehavior, muted: muted, onVideoFrame: onVideoFrame, playbackRate: playbackRate, src: src, style: style, volume: volume }));
 };
+export const Video = ({ src, audioStreamIndex, className, delayRenderRetries, delayRenderTimeoutInMilliseconds, disallowFallbackToOffthreadVideo, fallbackOffthreadVideoProps, logLevel, loop, loopVolumeCurveBehavior, muted, name, onVideoFrame, playbackRate, showInTimeline, style, trimAfter, trimBefore, volume, stack, }) => {
+    return (_jsx(InnerVideo, { audioStreamIndex: audioStreamIndex ?? 0, className: className, delayRenderRetries: delayRenderRetries ?? null, delayRenderTimeoutInMilliseconds: delayRenderTimeoutInMilliseconds ?? null, disallowFallbackToOffthreadVideo: disallowFallbackToOffthreadVideo ?? false, fallbackOffthreadVideoProps: fallbackOffthreadVideoProps ?? {}, logLevel: logLevel ?? 'info', loop: loop ?? false, loopVolumeCurveBehavior: loopVolumeCurveBehavior ?? 'repeat', muted: muted ?? false, name: name, onVideoFrame: onVideoFrame, playbackRate: playbackRate ?? 1, showInTimeline: showInTimeline ?? true, src: src, style: style ?? {}, trimAfter: trimAfter, trimBefore: trimBefore, volume: volume ?? 1, stack: stack }));
+};
+Internals.addSequenceStackTraces(Video);

package/dist/video-extraction/extract-frame-via-broadcast-channel.d.ts

@@ -1,6 +1,6 @@
 import { type LogLevel } from 'remotion';
 import type { PcmS16AudioData } from '../convert-audiodata/convert-audiodata';
-export declare const extractFrameViaBroadcastChannel: ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, isClientSideRendering, loop, }: {
+export declare const extractFrameViaBroadcastChannel: ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, isClientSideRendering, loop, audioStreamIndex, }: {
     src: string;
     timeInSeconds: number;
     durationInSeconds: number;
@@ -10,8 +10,9 @@ export declare const extractFrameViaBroadcastChannel: ({ src, timeInSeconds, log
     includeVideo: boolean;
     isClientSideRendering: boolean;
     loop: boolean;
+    audioStreamIndex: number;
 }) => Promise<{
     frame: ImageBitmap | VideoFrame | null;
     audio: PcmS16AudioData | null;
     durationInSeconds: number | null;
-}>;
+} | "cannot-decode" | "network-error" | "unknown-container-format">;

package/dist/video-extraction/extract-frame-via-broadcast-channel.js

@@ -5,7 +5,7 @@ if (window.remotion_broadcastChannel && window.remotion_isMainTab) {
         const data = event.data;
         if (data.type === 'request') {
             try {
-                const { frame, audio, durationInSeconds } = await extractFrameAndAudio({
+                const result = await extractFrameAndAudio({
                     src: data.src,
                     timeInSeconds: data.timeInSeconds,
                     logLevel: data.logLevel,
@@ -14,7 +14,33 @@ if (window.remotion_broadcastChannel && window.remotion_isMainTab) {
                     includeAudio: data.includeAudio,
                     includeVideo: data.includeVideo,
                     loop: data.loop,
+                    audioStreamIndex: data.audioStreamIndex,
                 });
+                if (result === 'cannot-decode') {
+                    const cannotDecodeResponse = {
+                        type: 'response-cannot-decode',
+                        id: data.id,
+                    };
+                    window.remotion_broadcastChannel.postMessage(cannotDecodeResponse);
+                    return;
+                }
+                if (result === 'network-error') {
+                    const networkErrorResponse = {
+                        type: 'response-network-error',
+                        id: data.id,
+                    };
+                    window.remotion_broadcastChannel.postMessage(networkErrorResponse);
+                    return;
+                }
+                if (result === 'unknown-container-format') {
+                    const unknownContainerFormatResponse = {
+                        type: 'response-unknown-container-format',
+                        id: data.id,
+                    };
+                    window.remotion_broadcastChannel.postMessage(unknownContainerFormatResponse);
+                    return;
+                }
+                const { frame, audio, durationInSeconds } = result;
                 const videoFrame = frame;
                 const imageBitmap = videoFrame
                     ? await createImageBitmap(videoFrame)
@@ -46,7 +72,7 @@ if (window.remotion_broadcastChannel && window.remotion_isMainTab) {
         }
     });
 }
-export const extractFrameViaBroadcastChannel = ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, isClientSideRendering, loop, }) => {
+export const extractFrameViaBroadcastChannel = ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, isClientSideRendering, loop, audioStreamIndex, }) => {
     if (isClientSideRendering || window.remotion_isMainTab) {
         return extractFrameAndAudio({
             logLevel,
@@ -57,6 +83,7 @@ export const extractFrameViaBroadcastChannel = ({ src, timeInSeconds, logLevel,
             includeAudio,
             includeVideo,
             loop,
+            audioStreamIndex,
         });
     }
     const requestId = crypto.randomUUID();
@@ -66,7 +93,10 @@ export const extractFrameViaBroadcastChannel = ({ src, timeInSeconds, logLevel,
             if (!data) {
                 return;
             }
-            if (data.type === 'response-success' && data.id === requestId) {
+            if (data.id !== requestId) {
+                return;
+            }
+            if (data.type === 'response-success') {
                 resolve({
                     frame: data.frame ? data.frame : null,
                     audio: data.audio ? data.audio : null,
@@ -75,11 +105,29 @@ export const extractFrameViaBroadcastChannel = ({ src, timeInSeconds, logLevel,
                         : null,
                 });
                 window.remotion_broadcastChannel.removeEventListener('message', onMessage);
+                return;
             }
-            else if (data.type === 'response-error' && data.id === requestId) {
+            if (data.type === 'response-error') {
                 reject(data.errorStack);
                 window.remotion_broadcastChannel.removeEventListener('message', onMessage);
+                return;
+            }
+            if (data.type === 'response-cannot-decode') {
+                resolve('cannot-decode');
+                window.remotion_broadcastChannel.removeEventListener('message', onMessage);
+                return;
+            }
+            if (data.type === 'response-network-error') {
+                resolve('network-error');
+                window.remotion_broadcastChannel.removeEventListener('message', onMessage);
+                return;
+            }
+            if (data.type === 'response-unknown-container-format') {
+                resolve('unknown-container-format');
+                window.remotion_broadcastChannel.removeEventListener('message', onMessage);
+                return;
             }
+            throw new Error(`Invalid message: ${JSON.stringify(data)}`);
         };
         window.remotion_broadcastChannel.addEventListener('message', onMessage);
     });
@@ -94,6 +142,7 @@ export const extractFrameViaBroadcastChannel = ({ src, timeInSeconds, logLevel,
         includeAudio,
         includeVideo,
         loop,
+        audioStreamIndex,
     };
     window.remotion_broadcastChannel.postMessage(request);
     let timeoutId;

package/dist/video-extraction/extract-frame.d.ts

@@ -1,9 +1,8 @@
+import type { VideoSample } from 'mediabunny';
 import { type LogLevel } from 'remotion';
-import { type GetSink } from './get-frames-since-keyframe';
-export declare const sinkPromises: Record<string, Promise<GetSink>>;
 export declare const extractFrame: ({ src, timeInSeconds: unloopedTimeinSeconds, logLevel, loop, }: {
     src: string;
     timeInSeconds: number;
     logLevel: LogLevel;
     loop: boolean;
-}) => Promise<import("mediabunny").VideoSample | null>;
+}) => Promise<VideoSample | "cannot-decode" | "unknown-container-format" | null>;

package/dist/video-extraction/extract-frame.js

@@ -1,16 +1,19 @@
 import { keyframeManager } from '../caches';
-import { getSinks } from './get-frames-since-keyframe';
-export const sinkPromises = {};
+import { getSinkWeak } from '../get-sink-weak';
 export const extractFrame = async ({ src, timeInSeconds: unloopedTimeinSeconds, logLevel, loop, }) => {
-    if (!sinkPromises[src]) {
-        sinkPromises[src] = getSinks(src);
-    }
-    const { video, getDuration } = await sinkPromises[src];
-    if (video === null) {
+    const sink = await getSinkWeak(src, logLevel);
+    const video = await sink.getVideo();
+    if (video === 'no-video-track') {
         throw new Error(`No video track found for ${src}`);
     }
+    if (video === 'cannot-decode') {
+        return 'cannot-decode';
+    }
+    if (video === 'unknown-container-format') {
+        return 'unknown-container-format';
+    }
     const timeInSeconds = loop
-        ? unloopedTimeinSeconds % (await getDuration())
+        ? unloopedTimeinSeconds % (await sink.getDuration())
         : unloopedTimeinSeconds;
     const keyframeBank = await keyframeManager.requestKeyframeBank({
         packetSink: video.packetSink,

package/dist/video-extraction/get-frames-since-keyframe.d.ts

@@ -1,23 +1,28 @@
 import type { EncodedPacket } from 'mediabunny';
 import { AudioSampleSink, EncodedPacketSink, VideoSampleSink } from 'mediabunny';
-export declare const getSinks: (src: string) => Promise<{
-    video: {
-        sampleSink: VideoSampleSink;
-        packetSink: EncodedPacketSink;
-    } | null;
-    audio: {
-        sampleSink: AudioSampleSink;
-    } | null;
+type VideoSinks = {
+    sampleSink: VideoSampleSink;
+    packetSink: EncodedPacketSink;
+};
+type AudioSinks = {
+    sampleSink: AudioSampleSink;
+};
+export type AudioSinkResult = AudioSinks | 'no-audio-track' | 'cannot-decode-audio' | 'unknown-container-format';
+export type VideoSinkResult = VideoSinks | 'no-video-track' | 'cannot-decode' | 'unknown-container-format';
+export declare const getSinks: (src: string) => Promise<WeakRef<{
+    getVideo: () => Promise<VideoSinkResult>;
+    getAudio: (index: number) => Promise<AudioSinkResult>;
     actualMatroskaTimestamps: {
         observeTimestamp: (startTime: number) => void;
         getRealTimestamp: (observedTimestamp: number) => number | null;
     };
     isMatroska: boolean;
     getDuration: () => Promise<number>;
-}>;
+}>>;
 export type GetSink = Awaited<ReturnType<typeof getSinks>>;
 export declare const getFramesSinceKeyframe: ({ packetSink, videoSampleSink, startPacket, }: {
     packetSink: EncodedPacketSink;
     videoSampleSink: VideoSampleSink;
     startPacket: EncodedPacket;
 }) => Promise<import("./keyframe-bank").KeyframeBank>;
+export {};