@remotion/media 4.0.364 → 4.0.365

package/dist/media-player.d.ts

@@ -37,9 +37,6 @@ export declare class MediaPlayer {
     private trimAfter;
     private initialized;
     private totalDuration;
-    private isBuffering;
-    private onBufferingChangeCallback?;
-    private mediaEnded;
     private debugOverlay;
     private onVideoFrameCallback?;
     private initializationPromise;
@@ -61,7 +58,6 @@ export declare class MediaPlayer {
     private input;
     private isReady;
     private hasAudio;
-    private isCurrentlyBuffering;
     private isDisposalError;
     initialize(startTimeUnresolved: number): Promise<MediaPlayerInitResult>;
     private _initialize;
@@ -70,7 +66,7 @@ export declare class MediaPlayer {
     private seekPromiseChain;
     seekTo(time: number): Promise<void>;
     seekToDoNotCallDirectly(time: number, nonce: number): Promise<void>;
-    play(): Promise<void>;
+    play(time: number): Promise<void>;
     pause(): void;
     setMuted(muted: boolean): void;
     setVolume(volume: number): void;
@@ -80,16 +76,12 @@ export declare class MediaPlayer {
     setLoop(loop: boolean): void;
     dispose(): Promise<void>;
     private getPlaybackTime;
+    private setPlaybackTime;
+    private audioChunksForAfterResuming;
     private scheduleAudioChunk;
-    onBufferingChange(callback: (isBuffering: boolean) => void): () => void;
     onVideoFrame(callback: (frame: CanvasImageSource) => void): () => void;
     private drawFrame;
     private startAudioIterator;
     private drawDebugOverlay;
     private startVideoIterator;
-    private bufferingStartedAtMs;
-    private minBufferingTimeoutMs;
-    private setBufferingState;
-    private maybeResumeFromBuffering;
-    private runAudioIterator;
 }

package/dist/media-player.js

@@ -1,18 +1,17 @@
 import { ALL_FORMATS, AudioBufferSink, CanvasSink, Input, UrlSource, } from 'mediabunny';
 import { Internals } from 'remotion';
-import { HEALTHY_BUFFER_THRESHOLD_SECONDS, makeAudioIterator, } from './audio/audio-preview-iterator';
+import { isAlreadyQueued, makeAudioIterator, } from './audio/audio-preview-iterator';
 import { drawPreviewOverlay } from './debug-overlay/preview-overlay';
 import { getTimeInSeconds } from './get-time-in-seconds';
 import { isNetworkError } from './is-network-error';
-import { sleep, TimeoutError, withTimeout } from './video/timeout-utils';
 import { createVideoIterator, } from './video/video-preview-iterator';
-const AUDIO_BUFFER_TOLERANCE_THRESHOLD = 0.1;
 export class MediaPlayer {
     constructor({ canvas, src, logLevel, sharedAudioContext, loop, trimBefore, trimAfter, playbackRate, audioStreamIndex, fps, debugOverlay, bufferState, }) {
         this.canvasSink = null;
         this.videoFrameIterator = null;
         this.debugStats = {
             videoIteratorsCreated: 0,
+            audioIteratorsCreated: 0,
             framesRendered: 0,
         };
         this.audioSink = null;
@@ -26,14 +25,12 @@ export class MediaPlayer {
         this.muted = false;
         this.loop = false;
         this.initialized = false;
-        // for remotion buffer state
-        this.isBuffering = false;
-        this.mediaEnded = false;
         this.debugOverlay = false;
         this.initializationPromise = null;
         this.input = null;
         this.currentSeekNonce = 0;
         this.seekPromiseChain = Promise.resolve();
+        this.audioChunksForAfterResuming = [];
         this.drawFrame = (frame) => {
             if (!this.context) {
                 throw new Error('Context not initialized');
@@ -47,22 +44,38 @@ export class MediaPlayer {
             }
             Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Drew frame ${frame.timestamp.toFixed(3)}s`);
         };
-        this.startAudioIterator = (startFromSecond) => {
+        this.startAudioIterator = async (startFromSecond, nonce) => {
             if (!this.hasAudio())
                 return;
-            // Clean up existing audio iterator
             this.audioBufferIterator?.destroy();
-            try {
-                const iterator = makeAudioIterator(this.audioSink, startFromSecond);
-                this.audioBufferIterator = iterator;
-                this.runAudioIterator(startFromSecond, iterator);
-            }
-            catch (error) {
-                if (this.isDisposalError()) {
+            this.audioChunksForAfterResuming = [];
+            const delayHandle = this.bufferState.delayPlayback();
+            const iterator = makeAudioIterator(this.audioSink, startFromSecond);
+            this.debugStats.audioIteratorsCreated++;
+            this.audioBufferIterator = iterator;
+            // Schedule up to 3 buffers ahead of the current time
+            for (let i = 0; i < 3; i++) {
+                const result = await iterator.getNext();
+                if (iterator.isDestroyed()) {
+                    delayHandle.unblock();
+                    return;
+                }
+                if (nonce !== this.currentSeekNonce) {
+                    delayHandle.unblock();
+                    return;
+                }
+                if (!result.value) {
+                    // media ended
+                    delayHandle.unblock();
                     return;
                 }
-                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start audio iterator', error);
+                const { buffer, timestamp } = result.value;
+                this.audioChunksForAfterResuming.push({
+                    buffer,
+                    timestamp,
+                });
             }
+            delayHandle.unblock();
         };
         this.startVideoIterator = async (timeToSeek, nonce) => {
             if (!this.canvasSink) {
@@ -72,9 +85,9 @@ export class MediaPlayer {
             const iterator = createVideoIterator(timeToSeek, this.canvasSink);
             this.debugStats.videoIteratorsCreated++;
             this.videoFrameIterator = iterator;
-            const delayHandle = this.bufferState?.delayPlayback();
+            const delayHandle = this.bufferState.delayPlayback();
             const frameResult = await iterator.getNext();
-            delayHandle?.unblock();
+            delayHandle.unblock();
             if (iterator.isDestroyed()) {
                 return;
             }
@@ -84,89 +97,11 @@ export class MediaPlayer {
             if (this.videoFrameIterator.isDestroyed()) {
                 return;
             }
-            if (frameResult.value) {
-                this.audioSyncAnchor =
-                    this.sharedAudioContext.currentTime - frameResult.value.timestamp;
-                this.drawFrame(frameResult.value);
-            }
-            else {
+            if (!frameResult.value) {
                 // media ended
-            }
-        };
-        this.bufferingStartedAtMs = null;
-        this.minBufferingTimeoutMs = 500;
-        this.runAudioIterator = async (startFromSecond, audioIterator) => {
-            if (!this.hasAudio())
                 return;
-            try {
-                let totalBufferDuration = 0;
-                let isFirstBuffer = true;
-                audioIterator.setAudioIteratorStarted(true);
-                while (true) {
-                    if (audioIterator.isDestroyed()) {
-                        return;
-                    }
-                    const BUFFERING_TIMEOUT_MS = 50;
-                    let result;
-                    try {
-                        result = await withTimeout(audioIterator.getNext(), BUFFERING_TIMEOUT_MS, 'Iterator timeout');
-                    }
-                    catch (error) {
-                        if (error instanceof TimeoutError && !this.mediaEnded) {
-                            this.setBufferingState(true);
-                        }
-                        await sleep(10);
-                        continue;
-                    }
-                    // media has ended
-                    if (result.done || !result.value) {
-                        this.mediaEnded = true;
-                        break;
-                    }
-                    const { buffer, timestamp, duration } = result.value;
-                    totalBufferDuration += duration;
-                    audioIterator.setAudioBufferHealth(Math.max(0, totalBufferDuration / this.playbackRate));
-                    this.maybeResumeFromBuffering(totalBufferDuration / this.playbackRate);
-                    if (this.playing) {
-                        if (isFirstBuffer) {
-                            this.audioSyncAnchor =
-                                this.sharedAudioContext.currentTime - timestamp;
-                            isFirstBuffer = false;
-                        }
-                        // if timestamp is less than timeToSeek, skip
-                        // context: for some reason, mediabunny returns buffer at 9.984s, when requested at 10s
-                        if (timestamp < startFromSecond - AUDIO_BUFFER_TOLERANCE_THRESHOLD) {
-                            continue;
-                        }
-                        this.scheduleAudioChunk(buffer, timestamp);
-                    }
-                    const playbackTime = this.getPlaybackTime();
-                    if (playbackTime === null) {
-                        continue;
-                    }
-                    if (timestamp - playbackTime >= 1) {
-                        await new Promise((resolve) => {
-                            const check = () => {
-                                const currentPlaybackTime = this.getPlaybackTime();
-                                if (currentPlaybackTime !== null &&
-                                    timestamp - currentPlaybackTime < 1) {
-                                    resolve();
-                                }
-                                else {
-                                    requestAnimationFrame(check);
-                                }
-                            };
-                            check();
-                        });
-                    }
-                }
-            }
-            catch (error) {
-                if (this.isDisposalError()) {
-                    return;
-                }
-                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to run audio iterator', error);
             }
+            this.drawFrame(frameResult.value);
         };
         this.canvas = canvas ?? null;
         this.src = src;
@@ -202,9 +137,6 @@ export class MediaPlayer {
     hasAudio() {
         return Boolean(this.audioSink && this.sharedAudioContext && this.gainNode);
     }
-    isCurrentlyBuffering() {
-        return this.isBuffering && Boolean(this.bufferingStartedAtMs);
-    }
     isDisposalError() {
         return this.input?.disposed === true;
     }
@@ -282,11 +214,12 @@ export class MediaPlayer {
                 return { type: 'success', durationInSeconds: this.totalDuration };
             }
             if (this.sharedAudioContext) {
-                this.audioSyncAnchor = this.sharedAudioContext.currentTime - startTime;
+                this.setPlaybackTime(startTime);
             }
             this.initialized = true;
             try {
-                this.startAudioIterator(startTime);
+                // intentionally not awaited
+                this.startAudioIterator(startTime, this.currentSeekNonce);
                 await this.startVideoIterator(startTime, this.currentSeekNonce);
             }
             catch (error) {
@@ -349,32 +282,95 @@ export class MediaPlayer {
         if (currentPlaybackTime === newTime) {
             return;
         }
-        const satisfyResult = await this.videoFrameIterator?.tryToSatisfySeek(newTime);
-        if (satisfyResult?.type === 'satisfied') {
-            this.drawFrame(satisfyResult.frame);
-            return;
+        const newAudioSyncAnchor = this.sharedAudioContext.currentTime - newTime;
+        const diff = Math.abs(newAudioSyncAnchor - this.audioSyncAnchor);
+        if (diff > 0.1) {
+            this.setPlaybackTime(newTime);
         }
-        if (this.currentSeekNonce !== nonce) {
-            return;
+        // Should return immediately, so it's okay to not use Promise.all here
+        const videoSatisfyResult = await this.videoFrameIterator?.tryToSatisfySeek(newTime);
+        if (videoSatisfyResult?.type === 'satisfied') {
+            this.drawFrame(videoSatisfyResult.frame);
+        }
+        else if (videoSatisfyResult && this.currentSeekNonce === nonce) {
+            this.startVideoIterator(newTime, nonce);
+        }
+        const queuedPeriod = this.audioBufferIterator?.getQueuedPeriod();
+        const currentTimeIsAlreadyQueued = isAlreadyQueued(newTime, queuedPeriod);
+        const toBeScheduled = [];
+        if (!currentTimeIsAlreadyQueued) {
+            const audioSatisfyResult = await this.audioBufferIterator?.tryToSatisfySeek(newTime);
+            if (this.currentSeekNonce !== nonce) {
+                return;
+            }
+            if (!audioSatisfyResult) {
+                return;
+            }
+            if (audioSatisfyResult.type === 'not-satisfied') {
+                await this.startAudioIterator(newTime, nonce);
+                return;
+            }
+            toBeScheduled.push(...audioSatisfyResult.buffers);
+        }
+        // TODO: What is this is beyond the end of the video
+        const nextTime = newTime +
+            // start of next frame
+            (1 / this.fps) * this.playbackRate +
+            // need the full duration of the next frame to be queued
+            (1 / this.fps) * this.playbackRate;
+        const nextIsAlreadyQueued = isAlreadyQueued(nextTime, queuedPeriod);
+        if (!nextIsAlreadyQueued) {
+            const audioSatisfyResult = await this.audioBufferIterator?.tryToSatisfySeek(nextTime);
+            if (this.currentSeekNonce !== nonce) {
+                return;
+            }
+            if (!audioSatisfyResult) {
+                return;
+            }
+            if (audioSatisfyResult.type === 'not-satisfied') {
+                await this.startAudioIterator(nextTime, nonce);
+                return;
+            }
+            toBeScheduled.push(...audioSatisfyResult.buffers);
+        }
+        for (const buffer of toBeScheduled) {
+            if (this.playing) {
+                this.scheduleAudioChunk(buffer.buffer, buffer.timestamp);
+            }
+            else {
+                this.audioChunksForAfterResuming.push({
+                    buffer: buffer.buffer,
+                    timestamp: buffer.timestamp,
+                });
+            }
         }
-        this.mediaEnded = false;
-        this.audioSyncAnchor = this.sharedAudioContext.currentTime - newTime;
-        this.startAudioIterator(newTime);
-        this.startVideoIterator(newTime, nonce);
     }
-    async play() {
+    async play(time) {
         if (!this.isReady())
             return;
-        if (!this.playing) {
-            if (this.sharedAudioContext.state === 'suspended') {
-                await this.sharedAudioContext.resume();
-            }
-            this.playing = true;
+        this.setPlaybackTime(time);
+        this.playing = true;
+        for (const chunk of this.audioChunksForAfterResuming) {
+            this.scheduleAudioChunk(chunk.buffer, chunk.timestamp);
         }
+        if (this.sharedAudioContext.state === 'suspended') {
+            await this.sharedAudioContext.resume();
+        }
+        this.audioChunksForAfterResuming.length = 0;
+        this.drawDebugOverlay();
     }
     pause() {
         this.playing = false;
-        this.audioBufferIterator?.cleanupAudioQueue();
+        const toQueue = this.audioBufferIterator?.removeAndReturnAllQueuedAudioNodes();
+        if (toQueue) {
+            for (const chunk of toQueue) {
+                this.audioChunksForAfterResuming.push({
+                    buffer: chunk.buffer,
+                    timestamp: chunk.timestamp,
+                });
+            }
+        }
+        this.drawDebugOverlay();
     }
     setMuted(muted) {
         this.muted = muted;
@@ -426,28 +422,27 @@ export class MediaPlayer {
     getPlaybackTime() {
         return this.sharedAudioContext.currentTime - this.audioSyncAnchor;
     }
+    setPlaybackTime(time) {
+        this.audioSyncAnchor = this.sharedAudioContext.currentTime - time;
+    }
     scheduleAudioChunk(buffer, mediaTimestamp) {
-        const targetTime = mediaTimestamp + this.audioSyncAnchor;
-        const delay = targetTime - this.sharedAudioContext.currentTime;
+        // TODO: Might already be scheduled, and then the playback rate changes
+        // TODO: Playbackrate does not yet work
+        const targetTime = (mediaTimestamp - (this.trimBefore ?? 0) / this.fps) / this.playbackRate;
+        const delay = targetTime + this.audioSyncAnchor - this.sharedAudioContext.currentTime;
         const node = this.sharedAudioContext.createBufferSource();
         node.buffer = buffer;
         node.playbackRate.value = this.playbackRate;
         node.connect(this.gainNode);
         if (delay >= 0) {
-            node.start(targetTime);
+            node.start(targetTime + this.audioSyncAnchor);
         }
         else {
             node.start(this.sharedAudioContext.currentTime, -delay);
         }
-        this.audioBufferIterator?.addQueuedAudioNode(node);
-        node.onended = () => this.audioBufferIterator?.removeQueuedAudioNode(node);
-    }
-    onBufferingChange(callback) {
-        this.onBufferingChangeCallback = callback;
-        return () => {
-            if (this.onBufferingChangeCallback === callback) {
-                this.onBufferingChangeCallback = undefined;
-            }
+        this.audioBufferIterator.addQueuedAudioNode(node, mediaTimestamp, buffer);
+        node.onended = () => {
+            return this.audioBufferIterator.removeQueuedAudioNode(node);
         };
     }
     onVideoFrame(callback) {
@@ -465,32 +460,16 @@ export class MediaPlayer {
         if (!this.debugOverlay)
             return;
         if (this.context && this.canvas) {
-            drawPreviewOverlay(this.context, this.debugStats, this.sharedAudioContext.state, this.sharedAudioContext.currentTime);
-        }
-    }
-    setBufferingState(isBuffering) {
-        if (this.isBuffering !== isBuffering) {
-            this.isBuffering = isBuffering;
-            if (isBuffering) {
-                this.bufferingStartedAtMs = performance.now();
-                this.onBufferingChangeCallback?.(true);
-            }
-            else {
-                this.bufferingStartedAtMs = null;
-                this.onBufferingChangeCallback?.(false);
-            }
-        }
-    }
-    maybeResumeFromBuffering(currentBufferDuration) {
-        if (!this.isCurrentlyBuffering())
-            return;
-        const now = performance.now();
-        const bufferingDuration = now - this.bufferingStartedAtMs;
-        const minTimeElapsed = bufferingDuration >= this.minBufferingTimeoutMs;
-        const bufferHealthy = currentBufferDuration >= HEALTHY_BUFFER_THRESHOLD_SECONDS;
-        if (minTimeElapsed && bufferHealthy) {
-            Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Resuming from buffering after ${bufferingDuration}ms - buffer recovered`);
-            this.setBufferingState(false);
+            drawPreviewOverlay({
+                context: this.context,
+                stats: this.debugStats,
+                audioTime: this.sharedAudioContext.currentTime,
+                audioContextState: this.sharedAudioContext.state,
+                audioSyncAnchor: this.audioSyncAnchor,
+                audioIterator: this.audioBufferIterator,
+                audioChunksForAfterResuming: this.audioChunksForAfterResuming,
+                playing: this.playing,
+            });
         }
     }
 }

package/dist/video/video-for-preview.js

@@ -60,6 +60,11 @@ export const VideoForPreview = ({ src: unpreloadedSrc, style, playbackRate, logL
     const currentTimeRef = useRef(currentTime);
     currentTimeRef.current = currentTime;
     const preloadedSrc = usePreload(src);
+    const buffering = useContext(Internals.BufferingContextReact);
+    if (!buffering) {
+        throw new Error('useMediaPlayback must be used inside a <BufferingContext>');
+    }
+    const isPlayerBuffering = Internals.useIsPlayerBuffering(buffering);
     useEffect(() => {
         if (!canvasRef.current)
             return;
@@ -167,15 +172,13 @@ export const VideoForPreview = ({ src: unpreloadedSrc, style, playbackRate, logL
         const mediaPlayer = mediaPlayerRef.current;
         if (!mediaPlayer)
             return;
-        if (playing) {
-            mediaPlayer.play().catch((error) => {
-                Internals.Log.error({ logLevel, tag: '@remotion/media' }, '[VideoForPreview] Failed to play', error);
-            });
+        if (playing && !isPlayerBuffering) {
+            mediaPlayer.play(currentTimeRef.current);
         }
         else {
             mediaPlayer.pause();
         }
-    }, [playing, logLevel, mediaPlayerReady]);
+    }, [isPlayerBuffering, playing, logLevel, mediaPlayerReady]);
     useLayoutEffect(() => {
         const mediaPlayer = mediaPlayerRef.current;
         if (!mediaPlayer || !mediaPlayerReady)
@@ -183,30 +186,6 @@ export const VideoForPreview = ({ src: unpreloadedSrc, style, playbackRate, logL
         mediaPlayer.seekTo(currentTime);
         Internals.Log.trace({ logLevel, tag: '@remotion/media' }, `[VideoForPreview] Updating target time to ${currentTime.toFixed(3)}s`);
     }, [currentTime, logLevel, mediaPlayerReady]);
-    useEffect(() => {
-        const mediaPlayer = mediaPlayerRef.current;
-        if (!mediaPlayer || !mediaPlayerReady)
-            return;
-        let currentBlock = null;
-        const unsubscribe = mediaPlayer.onBufferingChange((newBufferingState) => {
-            if (newBufferingState && !currentBlock) {
-                currentBlock = buffer.delayPlayback();
-                Internals.Log.trace({ logLevel, tag: '@remotion/media' }, '[VideoForPreview] MediaPlayer buffering - blocking Remotion playback');
-            }
-            else if (!newBufferingState && currentBlock) {
-                currentBlock.unblock();
-                currentBlock = null;
-                Internals.Log.trace({ logLevel, tag: '@remotion/media' }, '[VideoForPreview] MediaPlayer unbuffering - unblocking Remotion playback');
-            }
-        });
-        return () => {
-            unsubscribe();
-            if (currentBlock) {
-                currentBlock.unblock();
-                currentBlock = null;
-            }
-        };
-    }, [mediaPlayerReady, buffer, logLevel]);
     const effectiveMuted = isSequenceHidden || muted || mediaMuted || userPreferredVolume <= 0;
     useEffect(() => {
         const mediaPlayer = mediaPlayerRef.current;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@remotion/media",
-	"version": "4.0.364",
+	"version": "4.0.365",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
 	"module": "dist/esm/index.mjs",
@@ -21,8 +21,8 @@
 		"make": "tsc -d && bun --env-file=../.env.bundle bundle.ts"
 	},
 	"dependencies": {
-		"mediabunny": "1.24.1",
-		"remotion": "4.0.364",
+		"mediabunny": "1.24.2",
+		"remotion": "4.0.365",
 		"webdriverio": "9.19.2"
 	},
 	"peerDependencies": {
@@ -30,7 +30,7 @@
 		"react-dom": ">=16.8.0"
 	},
 	"devDependencies": {
-		"@remotion/eslint-config-internal": "4.0.364",
+		"@remotion/eslint-config-internal": "4.0.365",
 		"@vitest/browser": "^3.2.4",
 		"eslint": "9.19.0",
 		"react": "19.0.0",