@remotion/media 4.0.363 → 4.0.365

package/dist/audio/audio-for-preview.js

@@ -1,9 +1,9 @@
 import { jsx as _jsx } from "react/jsx-runtime";
 import { useContext, useEffect, useMemo, useRef, useState } from 'react';
 import { Internals, Audio as RemotionAudio, useBufferState, useCurrentFrame, } from 'remotion';
+import { MediaPlayer } from '../media-player';
 import { useLoopDisplay } from '../show-in-timeline';
 import { useMediaInTimeline } from '../use-media-in-timeline';
-import { MediaPlayer } from '../video/media-player';
 const { useUnsafeVideoConfig, Timeline, SharedAudioContext, useMediaMutedState, useMediaVolumeState, useFrameForVolumeProp, evaluateVolume, warnAboutTooHighVolume, usePreload, SequenceContext, } = Internals;
 const NewAudioForPreview = ({ src, playbackRate, logLevel, muted, volume, loopVolumeCurveBehavior, loop, trimAfter, trimBefore, name, showInTimeline, stack, disallowFallbackToHtml5Audio, toneFrequency, audioStreamIndex, fallbackHtml5AudioProps, }) => {
     const videoConfig = useUnsafeVideoConfig();
@@ -59,6 +59,11 @@ const NewAudioForPreview = ({ src, playbackRate, logLevel, muted, volume, loopVo
         trimAfter,
         trimBefore,
     });
+    const buffering = useContext(Internals.BufferingContextReact);
+    if (!buffering) {
+        throw new Error('useMediaPlayback must be used inside a <BufferingContext>');
+    }
+    const isPlayerBuffering = Internals.useIsPlayerBuffering(buffering);
     useEffect(() => {
         if (!sharedAudioContext)
             return;
@@ -76,6 +81,8 @@ const NewAudioForPreview = ({ src, playbackRate, logLevel, muted, volume, loopVo
                 canvas: null,
                 playbackRate,
                 audioStreamIndex: audioStreamIndex ?? 0,
+                debugOverlay: false,
+                bufferState: buffer,
             });
             mediaPlayerRef.current = player;
             player
@@ -152,20 +159,19 @@ const NewAudioForPreview = ({ src, playbackRate, logLevel, muted, volume, loopVo
         videoConfig.fps,
         audioStreamIndex,
         disallowFallbackToHtml5Audio,
+        buffer,
     ]);
     useEffect(() => {
         const audioPlayer = mediaPlayerRef.current;
         if (!audioPlayer)
             return;
-        if (playing) {
-            audioPlayer.play().catch((error) => {
-                Internals.Log.error({ logLevel, tag: '@remotion/media' }, '[NewAudioForPreview] Failed to play', error);
-            });
+        if (playing && !isPlayerBuffering) {
+            audioPlayer.play(currentTimeRef.current);
         }
         else {
             audioPlayer.pause();
         }
-    }, [playing, logLevel, mediaPlayerReady]);
+    }, [isPlayerBuffering, logLevel, playing]);
     useEffect(() => {
         const audioPlayer = mediaPlayerRef.current;
         if (!audioPlayer || !mediaPlayerReady)
@@ -173,22 +179,6 @@ const NewAudioForPreview = ({ src, playbackRate, logLevel, muted, volume, loopVo
         audioPlayer.seekTo(currentTime);
         Internals.Log.trace({ logLevel, tag: '@remotion/media' }, `[NewAudioForPreview] Updating target time to ${currentTime.toFixed(3)}s`);
     }, [currentTime, logLevel, mediaPlayerReady]);
-    useEffect(() => {
-        const audioPlayer = mediaPlayerRef.current;
-        if (!audioPlayer || !mediaPlayerReady)
-            return;
-        audioPlayer.onBufferingChange((newBufferingState) => {
-            if (newBufferingState && !delayHandleRef.current) {
-                delayHandleRef.current = buffer.delayPlayback();
-                Internals.Log.trace({ logLevel, tag: '@remotion/media' }, '[NewAudioForPreview] MediaPlayer buffering - blocking Remotion playback');
-            }
-            else if (!newBufferingState && delayHandleRef.current) {
-                delayHandleRef.current.unblock();
-                delayHandleRef.current = null;
-                Internals.Log.trace({ logLevel, tag: '@remotion/media' }, '[NewAudioForPreview] MediaPlayer unbuffering - unblocking Remotion playback');
-            }
-        });
-    }, [mediaPlayerReady, buffer, logLevel]);
     const effectiveMuted = muted || mediaMuted || userPreferredVolume <= 0;
     useEffect(() => {
         const audioPlayer = mediaPlayerRef.current;

package/dist/audio/audio-iterator.d.ts

@@ -0,0 +1,11 @@
+import type { AudioBufferSink } from 'mediabunny';
+export declare const HEALTHY_BUFFER_THRESHOLD_SECONDS = 1;
+export declare const makeAudioIterator: (audioSink: AudioBufferSink, startFromSecond: number) => {
+    iterator: AsyncGenerator<import("mediabunny").WrappedAudioBuffer, void, unknown>;
+    destroy: () => void;
+    isReadyToPlay: () => boolean;
+    setAudioIteratorStarted: (started: boolean) => void;
+    getNext: () => Promise<IteratorResult<import("mediabunny").WrappedAudioBuffer, void>>;
+    setAudioBufferHealth: (health: number) => void;
+};
+export type AudioIterator = ReturnType<typeof makeAudioIterator>;

package/dist/audio/audio-iterator.js

@@ -0,0 +1,24 @@
+export const HEALTHY_BUFFER_THRESHOLD_SECONDS = 1;
+export const makeAudioIterator = (audioSink, startFromSecond) => {
+    const iterator = audioSink.buffers(startFromSecond);
+    let audioIteratorStarted = false;
+    let audioBufferHealth = 0;
+    return {
+        iterator,
+        destroy: () => {
+            iterator.return().catch(() => undefined);
+        },
+        isReadyToPlay: () => {
+            return audioIteratorStarted && audioBufferHealth > 0;
+        },
+        setAudioIteratorStarted: (started) => {
+            audioIteratorStarted = started;
+        },
+        getNext: () => {
+            return iterator.next();
+        },
+        setAudioBufferHealth: (health) => {
+            audioBufferHealth = health;
+        },
+    };
+};

package/dist/audio/audio-preview-iterator.d.ts

@@ -0,0 +1,31 @@
+import type { AudioBufferSink, WrappedAudioBuffer } from 'mediabunny';
+export declare const HEALTHY_BUFFER_THRESHOLD_SECONDS = 1;
+export type QueuedNode = {
+    node: AudioBufferSourceNode;
+    timestamp: number;
+    buffer: AudioBuffer;
+};
+export declare const makeAudioIterator: (audioSink: AudioBufferSink, startFromSecond: number) => {
+    destroy: () => void;
+    getNext: () => Promise<IteratorResult<WrappedAudioBuffer, void>>;
+    isDestroyed: () => boolean;
+    addQueuedAudioNode: (node: AudioBufferSourceNode, timestamp: number, buffer: AudioBuffer) => void;
+    removeQueuedAudioNode: (node: AudioBufferSourceNode) => void;
+    removeAndReturnAllQueuedAudioNodes: () => QueuedNode[];
+    getQueuedPeriod: () => {
+        from: number;
+        until: number;
+    } | null;
+    tryToSatisfySeek: (time: number) => Promise<{
+        type: "not-satisfied";
+        reason: string;
+    } | {
+        type: "satisfied";
+        buffers: WrappedAudioBuffer[];
+    }>;
+};
+export type AudioIterator = ReturnType<typeof makeAudioIterator>;
+export declare const isAlreadyQueued: (time: number, queuedPeriod: {
+    from: number;
+    until: number;
+} | undefined | null) => boolean;

package/dist/audio/audio-preview-iterator.js

@@ -0,0 +1,168 @@
+import { roundTo4Digits } from '../helpers/round-to-4-digits';
+export const HEALTHY_BUFFER_THRESHOLD_SECONDS = 1;
+export const makeAudioIterator = (audioSink, startFromSecond) => {
+    let destroyed = false;
+    const iterator = audioSink.buffers(startFromSecond);
+    const queuedAudioNodes = [];
+    const cleanupAudioQueue = () => {
+        for (const node of queuedAudioNodes) {
+            node.node.stop();
+        }
+        queuedAudioNodes.length = 0;
+    };
+    let lastReturnedBuffer = null;
+    let iteratorEnded = false;
+    const getNextOrNullIfNotAvailable = async () => {
+        const next = iterator.next();
+        const result = await Promise.race([
+            next,
+            new Promise((resolve) => {
+                Promise.resolve().then(() => resolve());
+            }),
+        ]);
+        if (!result) {
+            return {
+                type: 'need-to-wait-for-it',
+                waitPromise: async () => {
+                    const res = await next;
+                    if (res.value) {
+                        lastReturnedBuffer = res.value;
+                    }
+                    else {
+                        iteratorEnded = true;
+                    }
+                    return res.value;
+                },
+            };
+        }
+        if (result.value) {
+            lastReturnedBuffer = result.value;
+        }
+        else {
+            iteratorEnded = true;
+        }
+        return {
+            type: 'got-buffer-or-end',
+            buffer: result.value ?? null,
+        };
+    };
+    const tryToSatisfySeek = async (time) => {
+        if (lastReturnedBuffer) {
+            const bufferTimestamp = roundTo4Digits(lastReturnedBuffer.timestamp);
+            const bufferEndTimestamp = roundTo4Digits(lastReturnedBuffer.timestamp + lastReturnedBuffer.duration);
+            if (roundTo4Digits(time) < bufferTimestamp) {
+                return {
+                    type: 'not-satisfied',
+                    reason: `iterator is too far, most recently returned ${bufferTimestamp}-${bufferEndTimestamp}, requested ${time}`,
+                };
+            }
+            if (roundTo4Digits(time) <= bufferEndTimestamp) {
+                return {
+                    type: 'satisfied',
+                    buffers: [lastReturnedBuffer],
+                };
+            }
+            // fall through
+        }
+        if (iteratorEnded) {
+            if (lastReturnedBuffer) {
+                return {
+                    type: 'satisfied',
+                    buffers: [lastReturnedBuffer],
+                };
+            }
+            return {
+                type: 'not-satisfied',
+                reason: 'iterator ended',
+            };
+        }
+        const toBeReturned = [];
+        while (true) {
+            const buffer = await getNextOrNullIfNotAvailable();
+            if (buffer.type === 'need-to-wait-for-it') {
+                return {
+                    type: 'not-satisfied',
+                    reason: 'iterator did not have buffer ready',
+                };
+            }
+            if (buffer.type === 'got-buffer-or-end') {
+                if (buffer.buffer === null) {
+                    iteratorEnded = true;
+                    if (lastReturnedBuffer) {
+                        return {
+                            type: 'satisfied',
+                            buffers: [lastReturnedBuffer],
+                        };
+                    }
+                    return {
+                        type: 'not-satisfied',
+                        reason: 'iterator ended and did not have buffer ready',
+                    };
+                }
+                const bufferTimestamp = roundTo4Digits(buffer.buffer.timestamp);
+                const bufferEndTimestamp = roundTo4Digits(buffer.buffer.timestamp + buffer.buffer.duration);
+                const timestamp = roundTo4Digits(time);
+                if (bufferTimestamp <= timestamp && bufferEndTimestamp > timestamp) {
+                    return {
+                        type: 'satisfied',
+                        buffers: [...toBeReturned, buffer.buffer],
+                    };
+                }
+                toBeReturned.push(buffer.buffer);
+                continue;
+            }
+            throw new Error('Unreachable');
+        }
+    };
+    return {
+        destroy: () => {
+            cleanupAudioQueue();
+            destroyed = true;
+            iterator.return().catch(() => undefined);
+        },
+        getNext: () => {
+            return iterator.next();
+        },
+        isDestroyed: () => {
+            return destroyed;
+        },
+        addQueuedAudioNode: (node, timestamp, buffer) => {
+            queuedAudioNodes.push({ node, timestamp, buffer });
+        },
+        removeQueuedAudioNode: (node) => {
+            const index = queuedAudioNodes.findIndex((n) => n.node === node);
+            if (index !== -1) {
+                queuedAudioNodes.splice(index, 1);
+            }
+        },
+        removeAndReturnAllQueuedAudioNodes: () => {
+            const nodes = queuedAudioNodes.slice();
+            for (const node of nodes) {
+                node.node.stop();
+            }
+            queuedAudioNodes.length = 0;
+            return nodes;
+        },
+        getQueuedPeriod: () => {
+            const lastNode = queuedAudioNodes[queuedAudioNodes.length - 1];
+            if (!lastNode) {
+                return null;
+            }
+            const firstNode = queuedAudioNodes[0];
+            if (!firstNode) {
+                return null;
+            }
+            return {
+                from: firstNode.timestamp,
+                until: lastNode.timestamp + lastNode.buffer.duration,
+            };
+        },
+        tryToSatisfySeek,
+    };
+};
+export const isAlreadyQueued = (time, queuedPeriod) => {
+    if (!queuedPeriod) {
+        return false;
+    }
+    return time >= queuedPeriod.from && time < queuedPeriod.until;
+};

package/dist/debug-overlay/preview-overlay.d.ts

@@ -0,0 +1,19 @@
+import type { AudioIterator } from '../audio/audio-preview-iterator';
+export type DebugStats = {
+    videoIteratorsCreated: number;
+    audioIteratorsCreated: number;
+    framesRendered: number;
+};
+export declare const drawPreviewOverlay: ({ context, stats, audioTime, audioContextState, audioIterator, audioSyncAnchor, audioChunksForAfterResuming, playing, }: {
+    context: CanvasRenderingContext2D;
+    stats: DebugStats;
+    audioTime: number;
+    audioContextState: AudioContextState;
+    audioSyncAnchor: number;
+    audioIterator: AudioIterator | null;
+    audioChunksForAfterResuming: {
+        buffer: AudioBuffer;
+        timestamp: number;
+    }[];
+    playing: boolean;
+}) => void;

package/dist/debug-overlay/preview-overlay.js

@@ -0,0 +1,37 @@
+export const drawPreviewOverlay = ({ context, stats, audioTime, audioContextState, audioIterator, audioSyncAnchor, audioChunksForAfterResuming, playing, }) => {
+    // Collect all lines to be rendered
+    const lines = [
+        'Debug overlay',
+        `Video iterators created: ${stats.videoIteratorsCreated}`,
+        `Audio iterators created: ${stats.audioIteratorsCreated}`,
+        `Frames rendered: ${stats.framesRendered}`,
+        `Audio context state: ${audioContextState}`,
+        `Audio time: ${(audioTime - audioSyncAnchor).toFixed(3)}s`,
+    ];
+    if (audioIterator) {
+        const queuedPeriod = audioIterator.getQueuedPeriod();
+        if (queuedPeriod) {
+            lines.push(`Audio queued until: ${(queuedPeriod.until - (audioTime - audioSyncAnchor)).toFixed(3)}s`);
+        }
+        else if (audioChunksForAfterResuming.length > 0) {
+            lines.push(`Audio chunks for after resuming: ${audioChunksForAfterResuming.length}`);
+        }
+        lines.push(`Playing: ${playing}`);
+    }
+    const lineHeight = 30; // px, should match or exceed font size
+    const boxPaddingX = 10;
+    const boxPaddingY = 10;
+    const boxLeft = 20;
+    const boxTop = 20;
+    const boxWidth = 600;
+    const boxHeight = lines.length * lineHeight + 2 * boxPaddingY;
+    // Draw background for text legibility
+    context.fillStyle = 'rgba(0, 0, 0, 1)';
+    context.fillRect(boxLeft, boxTop, boxWidth, boxHeight);
+    context.fillStyle = 'white';
+    context.font = '24px sans-serif';
+    context.textBaseline = 'top';
+    for (let i = 0; i < lines.length; i++) {
+        context.fillText(lines[i], boxLeft + boxPaddingX, boxTop + boxPaddingY + i * lineHeight);
+    }
+};