@remotion/media 4.0.365 → 4.0.366

package/dist/media-player.js

@@ -1,113 +1,65 @@
-import { ALL_FORMATS, AudioBufferSink, CanvasSink, Input, UrlSource, } from 'mediabunny';
+import { ALL_FORMATS, Input, UrlSource } from 'mediabunny';
 import { Internals } from 'remotion';
-import { isAlreadyQueued, makeAudioIterator, } from './audio/audio-preview-iterator';
+import { audioIteratorManager, } from './audio-iterator-manager';
+import { calculatePlaybackTime } from './calculate-playbacktime';
 import { drawPreviewOverlay } from './debug-overlay/preview-overlay';
 import { getTimeInSeconds } from './get-time-in-seconds';
 import { isNetworkError } from './is-network-error';
-import { createVideoIterator, } from './video/video-preview-iterator';
+import { makeNonceManager } from './nonce-manager';
+import { videoIteratorManager } from './video-iterator-manager';
 export class MediaPlayer {
-    constructor({ canvas, src, logLevel, sharedAudioContext, loop, trimBefore, trimAfter, playbackRate, audioStreamIndex, fps, debugOverlay, bufferState, }) {
-        this.canvasSink = null;
-        this.videoFrameIterator = null;
-        this.debugStats = {
-            videoIteratorsCreated: 0,
-            audioIteratorsCreated: 0,
-            framesRendered: 0,
-        };
-        this.audioSink = null;
-        this.audioBufferIterator = null;
-        this.gainNode = null;
-        this.currentVolume = 1;
+    constructor({ canvas, src, logLevel, sharedAudioContext, loop, trimBefore, trimAfter, playbackRate, globalPlaybackRate, audioStreamIndex, fps, debugOverlay, bufferState, isPremounting, isPostmounting, }) {
+        this.audioIteratorManager = null;
+        this.videoIteratorManager = null;
         // this is the time difference between Web Audio timeline
         // and media file timeline
         this.audioSyncAnchor = 0;
         this.playing = false;
-        this.muted = false;
         this.loop = false;
-        this.initialized = false;
         this.debugOverlay = false;
+        this.onVideoFrameCallback = null;
         this.initializationPromise = null;
-        this.input = null;
-        this.currentSeekNonce = 0;
         this.seekPromiseChain = Promise.resolve();
-        this.audioChunksForAfterResuming = [];
-        this.drawFrame = (frame) => {
-            if (!this.context) {
-                throw new Error('Context not initialized');
-            }
-            this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
-            this.context.drawImage(frame.canvas, 0, 0);
-            this.debugStats.framesRendered++;
-            this.drawDebugOverlay();
-            if (this.onVideoFrameCallback && this.canvas) {
-                this.onVideoFrameCallback(this.canvas);
+        this.delayPlaybackHandleIfNotPremounting = () => {
+            if (this.isPremounting || this.isPostmounting) {
+                return {
+                    unblock: () => { },
+                };
             }
-            Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Drew frame ${frame.timestamp.toFixed(3)}s`);
+            return this.bufferState.delayPlayback();
         };
-        this.startAudioIterator = async (startFromSecond, nonce) => {
-            if (!this.hasAudio())
-                return;
-            this.audioBufferIterator?.destroy();
-            this.audioChunksForAfterResuming = [];
-            const delayHandle = this.bufferState.delayPlayback();
-            const iterator = makeAudioIterator(this.audioSink, startFromSecond);
-            this.debugStats.audioIteratorsCreated++;
-            this.audioBufferIterator = iterator;
-            // Schedule up to 3 buffers ahead of the current time
-            for (let i = 0; i < 3; i++) {
-                const result = await iterator.getNext();
-                if (iterator.isDestroyed()) {
-                    delayHandle.unblock();
-                    return;
-                }
-                if (nonce !== this.currentSeekNonce) {
-                    delayHandle.unblock();
-                    return;
-                }
-                if (!result.value) {
-                    // media ended
-                    delayHandle.unblock();
-                    return;
-                }
-                const { buffer, timestamp } = result.value;
-                this.audioChunksForAfterResuming.push({
-                    buffer,
-                    timestamp,
-                });
+        this.scheduleAudioNode = (node, mediaTimestamp) => {
+            const currentTime = this.getPlaybackTime();
+            const delayWithoutPlaybackRate = mediaTimestamp - currentTime;
+            const delay = delayWithoutPlaybackRate / (this.playbackRate * this.globalPlaybackRate);
+            if (delay >= 0) {
+                node.start(this.sharedAudioContext.currentTime + delay);
             }
-            delayHandle.unblock();
-        };
-        this.startVideoIterator = async (timeToSeek, nonce) => {
-            if (!this.canvasSink) {
-                return;
-            }
-            this.videoFrameIterator?.destroy();
-            const iterator = createVideoIterator(timeToSeek, this.canvasSink);
-            this.debugStats.videoIteratorsCreated++;
-            this.videoFrameIterator = iterator;
-            const delayHandle = this.bufferState.delayPlayback();
-            const frameResult = await iterator.getNext();
-            delayHandle.unblock();
-            if (iterator.isDestroyed()) {
-                return;
-            }
-            if (nonce !== this.currentSeekNonce) {
-                return;
-            }
-            if (this.videoFrameIterator.isDestroyed()) {
-                return;
+            else {
+                node.start(this.sharedAudioContext.currentTime, -delay);
             }
-            if (!frameResult.value) {
-                // media ended
+        };
+        this.drawDebugOverlay = () => {
+            if (!this.debugOverlay)
                 return;
+            if (this.context && this.canvas) {
+                drawPreviewOverlay({
+                    context: this.context,
+                    audioTime: this.sharedAudioContext.currentTime,
+                    audioContextState: this.sharedAudioContext.state,
+                    audioSyncAnchor: this.audioSyncAnchor,
+                    audioIteratorManager: this.audioIteratorManager,
+                    playing: this.playing,
+                    videoIteratorManager: this.videoIteratorManager,
+                });
             }
-            this.drawFrame(frameResult.value);
         };
         this.canvas = canvas ?? null;
         this.src = src;
         this.logLevel = logLevel ?? window.remotion_logLevel;
         this.sharedAudioContext = sharedAudioContext;
         this.playbackRate = playbackRate;
+        this.globalPlaybackRate = globalPlaybackRate;
         this.loop = loop;
         this.trimBefore = trimBefore;
         this.trimAfter = trimAfter;
@@ -115,6 +67,13 @@ export class MediaPlayer {
         this.fps = fps;
         this.debugOverlay = debugOverlay;
         this.bufferState = bufferState;
+        this.isPremounting = isPremounting;
+        this.isPostmounting = isPostmounting;
+        this.nonceManager = makeNonceManager();
+        this.input = new Input({
+            source: new UrlSource(this.src),
+            formats: ALL_FORMATS,
+        });
         if (canvas) {
             const context = canvas.getContext('2d', {
                 alpha: true,
@@ -129,16 +88,8 @@ export class MediaPlayer {
             this.context = null;
         }
     }
-    isReady() {
-        return (this.initialized &&
-            Boolean(this.sharedAudioContext) &&
-            !this.input?.disposed);
-    }
-    hasAudio() {
-        return Boolean(this.audioSink && this.sharedAudioContext && this.gainNode);
-    }
     isDisposalError() {
-        return this.input?.disposed === true;
+        return this.input.disposed === true;
     }
     initialize(startTimeUnresolved) {
         const promise = this._initialize(startTimeUnresolved);
@@ -147,17 +98,11 @@ export class MediaPlayer {
     }
     async _initialize(startTimeUnresolved) {
         try {
-            const urlSource = new UrlSource(this.src);
-            const input = new Input({
-                source: urlSource,
-                formats: ALL_FORMATS,
-            });
-            this.input = input;
-            if (input.disposed) {
+            if (this.input.disposed) {
                 return { type: 'disposed' };
             }
             try {
-                await input.getFormat();
+                await this.input.getFormat();
             }
             catch (error) {
                 if (this.isDisposalError()) {
@@ -171,10 +116,13 @@ export class MediaPlayer {
                 return { type: 'unknown-container-format' };
             }
             const [durationInSeconds, videoTrack, audioTracks] = await Promise.all([
-                input.computeDuration(),
-                input.getPrimaryVideoTrack(),
-                input.getAudioTracks(),
+                this.input.computeDuration(),
+                this.input.getPrimaryVideoTrack(),
+                this.input.getAudioTracks(),
             ]);
+            if (this.input.disposed) {
+                return { type: 'disposed' };
+            }
             this.totalDuration = durationInSeconds;
             const audioTrack = audioTracks[this.audioStreamIndex] ?? null;
             if (!videoTrack && !audioTrack) {
@@ -185,18 +133,18 @@ export class MediaPlayer {
                 if (!canDecode) {
                     return { type: 'cannot-decode' };
                 }
-                this.canvasSink = new CanvasSink(videoTrack, {
-                    poolSize: 2,
-                    fit: 'contain',
-                    alpha: true,
+                if (this.input.disposed) {
+                    return { type: 'disposed' };
+                }
+                this.videoIteratorManager = videoIteratorManager({
+                    videoTrack,
+                    delayPlaybackHandleIfNotPremounting: this.delayPlaybackHandleIfNotPremounting,
+                    context: this.context,
+                    canvas: this.canvas,
+                    getOnVideoFrameCallback: () => this.onVideoFrameCallback,
+                    logLevel: this.logLevel,
+                    drawDebugOverlay: this.drawDebugOverlay,
                 });
-                this.canvas.width = videoTrack.displayWidth;
-                this.canvas.height = videoTrack.displayHeight;
-            }
-            if (audioTrack && this.sharedAudioContext) {
-                this.audioSink = new AudioBufferSink(audioTrack);
-                this.gainNode = this.sharedAudioContext.createGain();
-                this.gainNode.connect(this.sharedAudioContext.destination);
             }
             const startTime = getTimeInSeconds({
                 unloopedTimeInSeconds: startTimeUnresolved,
@@ -210,17 +158,29 @@ export class MediaPlayer {
                 src: this.src,
             });
             if (startTime === null) {
-                this.clearCanvas();
-                return { type: 'success', durationInSeconds: this.totalDuration };
-            }
-            if (this.sharedAudioContext) {
-                this.setPlaybackTime(startTime);
+                throw new Error(`should have asserted that the time is not null`);
+            }
+            this.setPlaybackTime(startTime, this.playbackRate * this.globalPlaybackRate);
+            if (audioTrack) {
+                this.audioIteratorManager = audioIteratorManager({
+                    audioTrack,
+                    delayPlaybackHandleIfNotPremounting: this.delayPlaybackHandleIfNotPremounting,
+                    sharedAudioContext: this.sharedAudioContext,
+                });
             }
-            this.initialized = true;
+            const nonce = this.nonceManager.createAsyncOperation();
             try {
                 // intentionally not awaited
-                this.startAudioIterator(startTime, this.currentSeekNonce);
-                await this.startVideoIterator(startTime, this.currentSeekNonce);
+                if (this.audioIteratorManager) {
+                    this.audioIteratorManager.startAudioIterator({
+                        nonce,
+                        playbackRate: this.playbackRate * this.globalPlaybackRate,
+                        startFromSecond: startTime,
+                        getIsPlaying: () => this.playing,
+                        scheduleAudioNode: this.scheduleAudioNode,
+                    });
+                }
+                await this.videoIteratorManager?.startVideoIterator(startTime, nonce);
             }
             catch (error) {
                 if (this.isDisposalError()) {
@@ -240,24 +200,7 @@ export class MediaPlayer {
             throw error;
         }
     }
-    clearCanvas() {
-        if (this.context && this.canvas) {
-            this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
-        }
-    }
     async seekTo(time) {
-        this.currentSeekNonce++;
-        const nonce = this.currentSeekNonce;
-        await this.seekPromiseChain;
-        this.seekPromiseChain = this.seekToDoNotCallDirectly(time, nonce);
-        await this.seekPromiseChain;
-    }
-    async seekToDoNotCallDirectly(time, nonce) {
-        if (nonce !== this.currentSeekNonce) {
-            return;
-        }
-        if (!this.isReady())
-            return;
         const newTime = getTimeInSeconds({
             unloopedTimeInSeconds: time,
             playbackRate: this.playbackRate,
@@ -270,138 +213,129 @@ export class MediaPlayer {
             src: this.src,
         });
         if (newTime === null) {
-            // invalidate in-flight video operations
-            this.videoFrameIterator?.destroy();
-            this.videoFrameIterator = null;
-            this.clearCanvas();
-            this.audioBufferIterator?.destroy();
-            this.audioBufferIterator = null;
+            throw new Error(`should have asserted that the time is not null`);
+        }
+        const nonce = this.nonceManager.createAsyncOperation();
+        await this.seekPromiseChain;
+        this.seekPromiseChain = this.seekToDoNotCallDirectly(newTime, nonce);
+        await this.seekPromiseChain;
+    }
+    async seekToDoNotCallDirectly(newTime, nonce) {
+        if (nonce.isStale()) {
             return;
         }
         const currentPlaybackTime = this.getPlaybackTime();
         if (currentPlaybackTime === newTime) {
             return;
         }
-        const newAudioSyncAnchor = this.sharedAudioContext.currentTime - newTime;
+        const newAudioSyncAnchor = this.sharedAudioContext.currentTime -
+            newTime / (this.playbackRate * this.globalPlaybackRate);
         const diff = Math.abs(newAudioSyncAnchor - this.audioSyncAnchor);
-        if (diff > 0.1) {
-            this.setPlaybackTime(newTime);
-        }
-        // Should return immediately, so it's okay to not use Promise.all here
-        const videoSatisfyResult = await this.videoFrameIterator?.tryToSatisfySeek(newTime);
-        if (videoSatisfyResult?.type === 'satisfied') {
-            this.drawFrame(videoSatisfyResult.frame);
-        }
-        else if (videoSatisfyResult && this.currentSeekNonce === nonce) {
-            this.startVideoIterator(newTime, nonce);
-        }
-        const queuedPeriod = this.audioBufferIterator?.getQueuedPeriod();
-        const currentTimeIsAlreadyQueued = isAlreadyQueued(newTime, queuedPeriod);
-        const toBeScheduled = [];
-        if (!currentTimeIsAlreadyQueued) {
-            const audioSatisfyResult = await this.audioBufferIterator?.tryToSatisfySeek(newTime);
-            if (this.currentSeekNonce !== nonce) {
-                return;
-            }
-            if (!audioSatisfyResult) {
-                return;
-            }
-            if (audioSatisfyResult.type === 'not-satisfied') {
-                await this.startAudioIterator(newTime, nonce);
-                return;
-            }
-            toBeScheduled.push(...audioSatisfyResult.buffers);
-        }
-        // TODO: What is this is beyond the end of the video
-        const nextTime = newTime +
-            // start of next frame
-            (1 / this.fps) * this.playbackRate +
-            // need the full duration of the next frame to be queued
-            (1 / this.fps) * this.playbackRate;
-        const nextIsAlreadyQueued = isAlreadyQueued(nextTime, queuedPeriod);
-        if (!nextIsAlreadyQueued) {
-            const audioSatisfyResult = await this.audioBufferIterator?.tryToSatisfySeek(nextTime);
-            if (this.currentSeekNonce !== nonce) {
-                return;
-            }
-            if (!audioSatisfyResult) {
-                return;
-            }
-            if (audioSatisfyResult.type === 'not-satisfied') {
-                await this.startAudioIterator(nextTime, nonce);
-                return;
-            }
-            toBeScheduled.push(...audioSatisfyResult.buffers);
-        }
-        for (const buffer of toBeScheduled) {
-            if (this.playing) {
-                this.scheduleAudioChunk(buffer.buffer, buffer.timestamp);
-            }
-            else {
-                this.audioChunksForAfterResuming.push({
-                    buffer: buffer.buffer,
-                    timestamp: buffer.timestamp,
-                });
-            }
+        if (diff > 0.04) {
+            this.setPlaybackTime(newTime, this.playbackRate * this.globalPlaybackRate);
         }
+        await this.videoIteratorManager?.seek({
+            newTime,
+            nonce,
+        });
+        await this.audioIteratorManager?.seek({
+            newTime,
+            nonce,
+            fps: this.fps,
+            playbackRate: this.playbackRate * this.globalPlaybackRate,
+            getIsPlaying: () => this.playing,
+            scheduleAudioNode: this.scheduleAudioNode,
+        });
     }
     async play(time) {
-        if (!this.isReady())
-            return;
-        this.setPlaybackTime(time);
+        const newTime = getTimeInSeconds({
+            unloopedTimeInSeconds: time,
+            playbackRate: this.playbackRate,
+            loop: this.loop,
+            trimBefore: this.trimBefore,
+            trimAfter: this.trimAfter,
+            mediaDurationInSeconds: this.totalDuration ?? null,
+            fps: this.fps,
+            ifNoMediaDuration: 'infinity',
+            src: this.src,
+        });
+        if (newTime === null) {
+            throw new Error(`should have asserted that the time is not null`);
+        }
+        this.setPlaybackTime(newTime, this.playbackRate * this.globalPlaybackRate);
         this.playing = true;
-        for (const chunk of this.audioChunksForAfterResuming) {
-            this.scheduleAudioChunk(chunk.buffer, chunk.timestamp);
+        if (this.audioIteratorManager) {
+            this.audioIteratorManager.resumeScheduledAudioChunks({
+                playbackRate: this.playbackRate * this.globalPlaybackRate,
+                scheduleAudioNode: this.scheduleAudioNode,
+            });
         }
         if (this.sharedAudioContext.state === 'suspended') {
             await this.sharedAudioContext.resume();
         }
-        this.audioChunksForAfterResuming.length = 0;
         this.drawDebugOverlay();
     }
     pause() {
         this.playing = false;
-        const toQueue = this.audioBufferIterator?.removeAndReturnAllQueuedAudioNodes();
-        if (toQueue) {
-            for (const chunk of toQueue) {
-                this.audioChunksForAfterResuming.push({
-                    buffer: chunk.buffer,
-                    timestamp: chunk.timestamp,
-                });
-            }
-        }
+        this.audioIteratorManager?.pausePlayback();
         this.drawDebugOverlay();
     }
     setMuted(muted) {
-        this.muted = muted;
-        if (this.gainNode) {
-            this.gainNode.gain.value = muted ? 0 : this.currentVolume;
-        }
+        this.audioIteratorManager?.setMuted(muted);
     }
     setVolume(volume) {
-        if (!this.gainNode) {
+        if (!this.audioIteratorManager) {
             return;
         }
-        const appliedVolume = Math.max(0, volume);
-        this.currentVolume = appliedVolume;
-        if (!this.muted) {
-            this.gainNode.gain.value = appliedVolume;
-        }
+        this.audioIteratorManager.setVolume(volume);
+    }
+    setTrimBefore(trimBefore) {
+        this.trimBefore = trimBefore;
+    }
+    setTrimAfter(trimAfter) {
+        this.trimAfter = trimAfter;
     }
     setDebugOverlay(debugOverlay) {
         this.debugOverlay = debugOverlay;
     }
+    updateAfterPlaybackRateChange() {
+        if (!this.audioIteratorManager) {
+            return;
+        }
+        this.setPlaybackTime(this.getPlaybackTime(), this.playbackRate * this.globalPlaybackRate);
+        const iterator = this.audioIteratorManager.getAudioBufferIterator();
+        if (!iterator) {
+            return;
+        }
+        iterator.moveQueuedChunksToPauseQueue();
+        if (this.playing) {
+            this.audioIteratorManager.resumeScheduledAudioChunks({
+                playbackRate: this.playbackRate * this.globalPlaybackRate,
+                scheduleAudioNode: this.scheduleAudioNode,
+            });
+        }
+    }
     setPlaybackRate(rate) {
         this.playbackRate = rate;
+        this.updateAfterPlaybackRateChange();
+    }
+    setGlobalPlaybackRate(rate) {
+        this.globalPlaybackRate = rate;
+        this.updateAfterPlaybackRateChange();
     }
     setFps(fps) {
         this.fps = fps;
     }
+    setIsPremounting(isPremounting) {
+        this.isPremounting = isPremounting;
+    }
+    setIsPostmounting(isPostmounting) {
+        this.isPostmounting = isPostmounting;
+    }
     setLoop(loop) {
         this.loop = loop;
     }
     async dispose() {
-        this.initialized = false;
         if (this.initializationPromise) {
             try {
                 // wait for the init to finished
@@ -413,63 +347,24 @@ export class MediaPlayer {
                 // Ignore initialization errors during disposal
             }
         }
-        this.input?.dispose();
-        this.videoFrameIterator?.destroy();
-        this.videoFrameIterator = null;
-        this.audioBufferIterator?.destroy();
-        this.audioBufferIterator = null;
+        // Mark all async operations as stale
+        this.nonceManager.createAsyncOperation();
+        this.videoIteratorManager?.destroy();
+        this.audioIteratorManager?.destroy();
+        this.input.dispose();
     }
     getPlaybackTime() {
-        return this.sharedAudioContext.currentTime - this.audioSyncAnchor;
-    }
-    setPlaybackTime(time) {
-        this.audioSyncAnchor = this.sharedAudioContext.currentTime - time;
+        return calculatePlaybackTime({
+            audioSyncAnchor: this.audioSyncAnchor,
+            currentTime: this.sharedAudioContext.currentTime,
+            playbackRate: this.playbackRate * this.globalPlaybackRate,
+        });
     }
-    scheduleAudioChunk(buffer, mediaTimestamp) {
-        // TODO: Might already be scheduled, and then the playback rate changes
-        // TODO: Playbackrate does not yet work
-        const targetTime = (mediaTimestamp - (this.trimBefore ?? 0) / this.fps) / this.playbackRate;
-        const delay = targetTime + this.audioSyncAnchor - this.sharedAudioContext.currentTime;
-        const node = this.sharedAudioContext.createBufferSource();
-        node.buffer = buffer;
-        node.playbackRate.value = this.playbackRate;
-        node.connect(this.gainNode);
-        if (delay >= 0) {
-            node.start(targetTime + this.audioSyncAnchor);
-        }
-        else {
-            node.start(this.sharedAudioContext.currentTime, -delay);
-        }
-        this.audioBufferIterator.addQueuedAudioNode(node, mediaTimestamp, buffer);
-        node.onended = () => {
-            return this.audioBufferIterator.removeQueuedAudioNode(node);
-        };
+    setPlaybackTime(time, playbackRate) {
+        this.audioSyncAnchor =
+            this.sharedAudioContext.currentTime - time / playbackRate;
     }
-    onVideoFrame(callback) {
+    setVideoFrameCallback(callback) {
         this.onVideoFrameCallback = callback;
-        if (this.initialized && callback && this.canvas) {
-            callback(this.canvas);
-        }
-        return () => {
-            if (this.onVideoFrameCallback === callback) {
-                this.onVideoFrameCallback = undefined;
-            }
-        };
-    }
-    drawDebugOverlay() {
-        if (!this.debugOverlay)
-            return;
-        if (this.context && this.canvas) {
-            drawPreviewOverlay({
-                context: this.context,
-                stats: this.debugStats,
-                audioTime: this.sharedAudioContext.currentTime,
-                audioContextState: this.sharedAudioContext.state,
-                audioSyncAnchor: this.audioSyncAnchor,
-                audioIterator: this.audioBufferIterator,
-                audioChunksForAfterResuming: this.audioChunksForAfterResuming,
-                playing: this.playing,
-            });
-        }
     }
 }

package/dist/nonce-manager.d.ts

@@ -0,0 +1,9 @@
+export type Nonce = {
+    isStale: () => boolean;
+};
+export type NonceManager = ReturnType<typeof makeNonceManager>;
+export declare const makeNonceManager: () => {
+    createAsyncOperation: () => {
+        isStale: () => boolean;
+    };
+};

package/dist/nonce-manager.js

@@ -0,0 +1,13 @@
+export const makeNonceManager = () => {
+    let nonce = 0;
+    const createAsyncOperation = () => {
+        nonce++;
+        const currentNonce = nonce;
+        return {
+            isStale: () => nonce !== currentNonce,
+        };
+    };
+    return {
+        createAsyncOperation,
+    };
+};