@remotion/media 4.0.364 → 4.0.365

package/dist/audio/audio-for-preview.js

@@ -59,6 +59,11 @@ const NewAudioForPreview = ({ src, playbackRate, logLevel, muted, volume, loopVo
         trimAfter,
         trimBefore,
     });
+    const buffering = useContext(Internals.BufferingContextReact);
+    if (!buffering) {
+        throw new Error('useMediaPlayback must be used inside a <BufferingContext>');
+    }
+    const isPlayerBuffering = Internals.useIsPlayerBuffering(buffering);
     useEffect(() => {
         if (!sharedAudioContext)
             return;
@@ -160,15 +165,13 @@ const NewAudioForPreview = ({ src, playbackRate, logLevel, muted, volume, loopVo
         const audioPlayer = mediaPlayerRef.current;
         if (!audioPlayer)
             return;
-        if (playing) {
-            audioPlayer.play().catch((error) => {
-                Internals.Log.error({ logLevel, tag: '@remotion/media' }, '[NewAudioForPreview] Failed to play', error);
-            });
+        if (playing && !isPlayerBuffering) {
+            audioPlayer.play(currentTimeRef.current);
         }
         else {
             audioPlayer.pause();
         }
-    }, [playing, logLevel, mediaPlayerReady]);
+    }, [isPlayerBuffering, logLevel, playing]);
     useEffect(() => {
         const audioPlayer = mediaPlayerRef.current;
         if (!audioPlayer || !mediaPlayerReady)
@@ -176,22 +179,6 @@ const NewAudioForPreview = ({ src, playbackRate, logLevel, muted, volume, loopVo
         audioPlayer.seekTo(currentTime);
         Internals.Log.trace({ logLevel, tag: '@remotion/media' }, `[NewAudioForPreview] Updating target time to ${currentTime.toFixed(3)}s`);
     }, [currentTime, logLevel, mediaPlayerReady]);
-    useEffect(() => {
-        const audioPlayer = mediaPlayerRef.current;
-        if (!audioPlayer || !mediaPlayerReady)
-            return;
-        audioPlayer.onBufferingChange((newBufferingState) => {
-            if (newBufferingState && !delayHandleRef.current) {
-                delayHandleRef.current = buffer.delayPlayback();
-                Internals.Log.trace({ logLevel, tag: '@remotion/media' }, '[NewAudioForPreview] MediaPlayer buffering - blocking Remotion playback');
-            }
-            else if (!newBufferingState && delayHandleRef.current) {
-                delayHandleRef.current.unblock();
-                delayHandleRef.current = null;
-                Internals.Log.trace({ logLevel, tag: '@remotion/media' }, '[NewAudioForPreview] MediaPlayer unbuffering - unblocking Remotion playback');
-            }
-        });
-    }, [mediaPlayerReady, buffer, logLevel]);
     const effectiveMuted = muted || mediaMuted || userPreferredVolume <= 0;
     useEffect(() => {
         const audioPlayer = mediaPlayerRef.current;

package/dist/audio/audio-preview-iterator.d.ts

@@ -1,14 +1,31 @@
-import type { AudioBufferSink } from 'mediabunny';
+import type { AudioBufferSink, WrappedAudioBuffer } from 'mediabunny';
 export declare const HEALTHY_BUFFER_THRESHOLD_SECONDS = 1;
+export type QueuedNode = {
+    node: AudioBufferSourceNode;
+    timestamp: number;
+    buffer: AudioBuffer;
+};
 export declare const makeAudioIterator: (audioSink: AudioBufferSink, startFromSecond: number) => {
-    cleanupAudioQueue: () => void;
     destroy: () => void;
-    isReadyToPlay: () => boolean;
-    setAudioIteratorStarted: (started: boolean) => void;
-    getNext: () => Promise<IteratorResult<import("mediabunny").WrappedAudioBuffer, void>>;
-    setAudioBufferHealth: (health: number) => void;
+    getNext: () => Promise<IteratorResult<WrappedAudioBuffer, void>>;
     isDestroyed: () => boolean;
-    addQueuedAudioNode: (node: AudioBufferSourceNode) => void;
+    addQueuedAudioNode: (node: AudioBufferSourceNode, timestamp: number, buffer: AudioBuffer) => void;
     removeQueuedAudioNode: (node: AudioBufferSourceNode) => void;
+    removeAndReturnAllQueuedAudioNodes: () => QueuedNode[];
+    getQueuedPeriod: () => {
+        from: number;
+        until: number;
+    } | null;
+    tryToSatisfySeek: (time: number) => Promise<{
+        type: "not-satisfied";
+        reason: string;
+    } | {
+        type: "satisfied";
+        buffers: WrappedAudioBuffer[];
+    }>;
 };
 export type AudioIterator = ReturnType<typeof makeAudioIterator>;
+export declare const isAlreadyQueued: (time: number, queuedPeriod: {
+    from: number;
+    until: number;
+} | undefined | null) => boolean;

package/dist/audio/audio-preview-iterator.js

@@ -1,43 +1,168 @@
+import { roundTo4Digits } from '../helpers/round-to-4-digits';
 export const HEALTHY_BUFFER_THRESHOLD_SECONDS = 1;
 export const makeAudioIterator = (audioSink, startFromSecond) => {
     let destroyed = false;
     const iterator = audioSink.buffers(startFromSecond);
-    let audioIteratorStarted = false;
-    let audioBufferHealth = 0;
-    const queuedAudioNodes = new Set();
+    const queuedAudioNodes = [];
     const cleanupAudioQueue = () => {
         for (const node of queuedAudioNodes) {
-            node.stop();
+            node.node.stop();
+        }
+        queuedAudioNodes.length = 0;
+    };
+    let lastReturnedBuffer = null;
+    let iteratorEnded = false;
+    const getNextOrNullIfNotAvailable = async () => {
+        const next = iterator.next();
+        const result = await Promise.race([
+            next,
+            new Promise((resolve) => {
+                Promise.resolve().then(() => resolve());
+            }),
+        ]);
+        if (!result) {
+            return {
+                type: 'need-to-wait-for-it',
+                waitPromise: async () => {
+                    const res = await next;
+                    if (res.value) {
+                        lastReturnedBuffer = res.value;
+                    }
+                    else {
+                        iteratorEnded = true;
+                    }
+                    return res.value;
+                },
+            };
+        }
+        if (result.value) {
+            lastReturnedBuffer = result.value;
+        }
+        else {
+            iteratorEnded = true;
+        }
+        return {
+            type: 'got-buffer-or-end',
+            buffer: result.value ?? null,
+        };
+    };
+    const tryToSatisfySeek = async (time) => {
+        if (lastReturnedBuffer) {
+            const bufferTimestamp = roundTo4Digits(lastReturnedBuffer.timestamp);
+            const bufferEndTimestamp = roundTo4Digits(lastReturnedBuffer.timestamp + lastReturnedBuffer.duration);
+            if (roundTo4Digits(time) < bufferTimestamp) {
+                return {
+                    type: 'not-satisfied',
+                    reason: `iterator is too far, most recently returned ${bufferTimestamp}-${bufferEndTimestamp}, requested ${time}`,
+                };
+            }
+            if (roundTo4Digits(time) <= bufferEndTimestamp) {
+                return {
+                    type: 'satisfied',
+                    buffers: [lastReturnedBuffer],
+                };
+            }
+            // fall through
+        }
+        if (iteratorEnded) {
+            if (lastReturnedBuffer) {
+                return {
+                    type: 'satisfied',
+                    buffers: [lastReturnedBuffer],
+                };
+            }
+            return {
+                type: 'not-satisfied',
+                reason: 'iterator ended',
+            };
+        }
+        const toBeReturned = [];
+        while (true) {
+            const buffer = await getNextOrNullIfNotAvailable();
+            if (buffer.type === 'need-to-wait-for-it') {
+                return {
+                    type: 'not-satisfied',
+                    reason: 'iterator did not have buffer ready',
+                };
+            }
+            if (buffer.type === 'got-buffer-or-end') {
+                if (buffer.buffer === null) {
+                    iteratorEnded = true;
+                    if (lastReturnedBuffer) {
+                        return {
+                            type: 'satisfied',
+                            buffers: [lastReturnedBuffer],
+                        };
+                    }
+                    return {
+                        type: 'not-satisfied',
+                        reason: 'iterator ended and did not have buffer ready',
+                    };
+                }
+                const bufferTimestamp = roundTo4Digits(buffer.buffer.timestamp);
+                const bufferEndTimestamp = roundTo4Digits(buffer.buffer.timestamp + buffer.buffer.duration);
+                const timestamp = roundTo4Digits(time);
+                if (bufferTimestamp <= timestamp && bufferEndTimestamp > timestamp) {
+                    return {
+                        type: 'satisfied',
+                        buffers: [...toBeReturned, buffer.buffer],
+                    };
+                }
+                toBeReturned.push(buffer.buffer);
+                continue;
+            }
+            throw new Error('Unreachable');
         }
-        queuedAudioNodes.clear();
     };
     return {
-        cleanupAudioQueue,
         destroy: () => {
             cleanupAudioQueue();
             destroyed = true;
             iterator.return().catch(() => undefined);
         },
-        isReadyToPlay: () => {
-            return audioIteratorStarted && audioBufferHealth > 0;
-        },
-        setAudioIteratorStarted: (started) => {
-            audioIteratorStarted = started;
-        },
         getNext: () => {
             return iterator.next();
         },
-        setAudioBufferHealth: (health) => {
-            audioBufferHealth = health;
-        },
         isDestroyed: () => {
             return destroyed;
         },
-        addQueuedAudioNode: (node) => {
-            queuedAudioNodes.add(node);
+        addQueuedAudioNode: (node, timestamp, buffer) => {
+            queuedAudioNodes.push({ node, timestamp, buffer });
         },
         removeQueuedAudioNode: (node) => {
-            queuedAudioNodes.delete(node);
+            const index = queuedAudioNodes.findIndex((n) => n.node === node);
+            if (index !== -1) {
+                queuedAudioNodes.splice(index, 1);
+            }
         },
+        removeAndReturnAllQueuedAudioNodes: () => {
+            const nodes = queuedAudioNodes.slice();
+            for (const node of nodes) {
+                node.node.stop();
+            }
+            queuedAudioNodes.length = 0;
+            return nodes;
+        },
+        getQueuedPeriod: () => {
+            const lastNode = queuedAudioNodes[queuedAudioNodes.length - 1];
+            if (!lastNode) {
+                return null;
+            }
+            const firstNode = queuedAudioNodes[0];
+            if (!firstNode) {
+                return null;
+            }
+            return {
+                from: firstNode.timestamp,
+                until: lastNode.timestamp + lastNode.buffer.duration,
+            };
+        },
+        tryToSatisfySeek,
     };
 };
+export const isAlreadyQueued = (time, queuedPeriod) => {
+    if (!queuedPeriod) {
+        return false;
+    }
+    return time >= queuedPeriod.from && time < queuedPeriod.until;
+};

package/dist/debug-overlay/preview-overlay.d.ts

@@ -1,5 +1,19 @@
+import type { AudioIterator } from '../audio/audio-preview-iterator';
 export type DebugStats = {
     videoIteratorsCreated: number;
+    audioIteratorsCreated: number;
     framesRendered: number;
 };
-export declare const drawPreviewOverlay: (context: CanvasRenderingContext2D, stats: DebugStats, audioContextState: AudioContextState, audioSyncAnchor: number) => void;
+export declare const drawPreviewOverlay: ({ context, stats, audioTime, audioContextState, audioIterator, audioSyncAnchor, audioChunksForAfterResuming, playing, }: {
+    context: CanvasRenderingContext2D;
+    stats: DebugStats;
+    audioTime: number;
+    audioContextState: AudioContextState;
+    audioSyncAnchor: number;
+    audioIterator: AudioIterator | null;
+    audioChunksForAfterResuming: {
+        buffer: AudioBuffer;
+        timestamp: number;
+    }[];
+    playing: boolean;
+}) => void;

package/dist/debug-overlay/preview-overlay.js

@@ -1,13 +1,37 @@
-export const drawPreviewOverlay = (context, stats, audioContextState, audioSyncAnchor) => {
-    // Optionally, set a background for text legibility
+export const drawPreviewOverlay = ({ context, stats, audioTime, audioContextState, audioIterator, audioSyncAnchor, audioChunksForAfterResuming, playing, }) => {
+    // Collect all lines to be rendered
+    const lines = [
+        'Debug overlay',
+        `Video iterators created: ${stats.videoIteratorsCreated}`,
+        `Audio iterators created: ${stats.audioIteratorsCreated}`,
+        `Frames rendered: ${stats.framesRendered}`,
+        `Audio context state: ${audioContextState}`,
+        `Audio time: ${(audioTime - audioSyncAnchor).toFixed(3)}s`,
+    ];
+    if (audioIterator) {
+        const queuedPeriod = audioIterator.getQueuedPeriod();
+        if (queuedPeriod) {
+            lines.push(`Audio queued until: ${(queuedPeriod.until - (audioTime - audioSyncAnchor)).toFixed(3)}s`);
+        }
+        else if (audioChunksForAfterResuming.length > 0) {
+            lines.push(`Audio chunks for after resuming: ${audioChunksForAfterResuming.length}`);
+        }
+        lines.push(`Playing: ${playing}`);
+    }
+    const lineHeight = 30; // px, should match or exceed font size
+    const boxPaddingX = 10;
+    const boxPaddingY = 10;
+    const boxLeft = 20;
+    const boxTop = 20;
+    const boxWidth = 600;
+    const boxHeight = lines.length * lineHeight + 2 * boxPaddingY;
+    // Draw background for text legibility
     context.fillStyle = 'rgba(0, 0, 0, 1)';
-    context.fillRect(20, 20, 600, 180);
+    context.fillRect(boxLeft, boxTop, boxWidth, boxHeight);
     context.fillStyle = 'white';
     context.font = '24px sans-serif';
     context.textBaseline = 'top';
-    context.fillText(`Debug overlay`, 30, 30);
-    context.fillText(`Video iterators created: ${stats.videoIteratorsCreated}`, 30, 60);
-    context.fillText(`Frames rendered: ${stats.framesRendered}`, 30, 90);
-    context.fillText(`Audio context state: ${audioContextState}`, 30, 120);
-    context.fillText(`Audio time: ${audioSyncAnchor.toFixed(3)}s`, 30, 150);
+    for (let i = 0; i < lines.length; i++) {
+        context.fillText(lines[i], boxLeft + boxPaddingX, boxTop + boxPaddingY + i * lineHeight);
+    }
 };