@remotion/media 4.0.390 → 4.0.392

package/dist/video/media-player.js

@@ -0,0 +1,361 @@
+import { ALL_FORMATS, AudioBufferSink, CanvasSink, Input, UrlSource, } from 'mediabunny';
+import { Internals } from 'remotion';
+import { sleep, withTimeout } from './timeout-utils';
+export const SEEK_THRESHOLD = 0.05;
+export class MediaPlayer {
+    constructor({ canvas, src, logLevel, sharedAudioContext, }) {
+        this.canvasSink = null;
+        this.videoFrameIterator = null;
+        this.nextFrame = null;
+        this.audioSink = null;
+        this.audioBufferIterator = null;
+        this.queuedAudioNodes = new Set();
+        this.gainNode = null;
+        // audioDelay = mediaTimestamp + audioSyncAnchor - sharedAudioContext.currentTime
+        this.audioSyncAnchor = 0;
+        this.playing = false;
+        this.animationFrameId = null;
+        this.videoAsyncId = 0;
+        this.initialized = false;
+        this.totalDuration = 0;
+        // for remotion buffer state
+        this.isBuffering = false;
+        this.audioBufferHealth = 0;
+        this.audioIteratorStarted = false;
+        this.HEALTHY_BUFER_THRESHOLD_SECONDS = 1;
+        this.input = null;
+        this.render = () => {
+            if (this.isBuffering) {
+                this.maybeForceResumeFromBuffering();
+            }
+            if (this.shouldRenderFrame()) {
+                this.drawCurrentFrame();
+            }
+            if (this.playing) {
+                this.animationFrameId = requestAnimationFrame(this.render);
+            }
+            else {
+                this.animationFrameId = null;
+            }
+        };
+        this.startAudioIterator = async (timeToSeek) => {
+            if (!this.hasAudio())
+                return;
+            // Clean up existing audio iterator
+            await this.audioBufferIterator?.return();
+            this.audioIteratorStarted = false;
+            this.audioBufferHealth = 0;
+            try {
+                this.audioBufferIterator = this.audioSink.buffers(timeToSeek);
+                this.runAudioIterator();
+            }
+            catch (error) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start audio iterator', error);
+            }
+        };
+        this.startVideoIterator = async (timeToSeek) => {
+            if (!this.canvasSink) {
+                return;
+            }
+            this.videoAsyncId++;
+            const currentAsyncId = this.videoAsyncId;
+            await this.videoFrameIterator?.return();
+            this.videoFrameIterator = this.canvasSink.canvases(timeToSeek);
+            try {
+                const firstFrame = (await this.videoFrameIterator.next()).value ?? null;
+                const secondFrame = (await this.videoFrameIterator.next()).value ?? null;
+                if (currentAsyncId !== this.videoAsyncId) {
+                    return;
+                }
+                if (firstFrame) {
+                    Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Drew initial frame ${firstFrame.timestamp.toFixed(3)}s`);
+                    this.context.drawImage(firstFrame.canvas, 0, 0);
+                }
+                this.nextFrame = secondFrame ?? null;
+                if (secondFrame) {
+                    Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Buffered next frame ${secondFrame.timestamp.toFixed(3)}s`);
+                }
+            }
+            catch (error) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start video iterator', error);
+            }
+        };
+        this.updateNextFrame = async () => {
+            if (!this.videoFrameIterator) {
+                return;
+            }
+            try {
+                while (true) {
+                    const newNextFrame = (await this.videoFrameIterator.next()).value ?? null;
+                    if (!newNextFrame) {
+                        break;
+                    }
+                    if (newNextFrame.timestamp <= this.getPlaybackTime()) {
+                        continue;
+                    }
+                    else {
+                        this.nextFrame = newNextFrame;
+                        Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Buffered next frame ${newNextFrame.timestamp.toFixed(3)}s`);
+                        break;
+                    }
+                }
+            }
+            catch (error) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to update next frame', error);
+            }
+        };
+        this.bufferingStartedAtMs = null;
+        this.minBufferingTimeoutMs = 500;
+        this.runAudioIterator = async () => {
+            if (!this.hasAudio() || !this.audioBufferIterator)
+                return;
+            try {
+                let totalBufferDuration = 0;
+                let isFirstBuffer = true;
+                this.audioIteratorStarted = true;
+                while (true) {
+                    const BUFFERING_TIMEOUT_MS = 50;
+                    let result;
+                    try {
+                        result = await withTimeout(this.audioBufferIterator.next(), BUFFERING_TIMEOUT_MS, 'Iterator timeout');
+                    }
+                    catch {
+                        this.setBufferingState(true);
+                        await sleep(10);
+                        continue;
+                    }
+                    if (result.done || !result.value) {
+                        break;
+                    }
+                    const { buffer, timestamp, duration } = result.value;
+                    totalBufferDuration += duration;
+                    this.audioBufferHealth = Math.max(0, totalBufferDuration);
+                    this.maybeResumeFromBuffering(totalBufferDuration);
+                    if (this.playing) {
+                        if (isFirstBuffer) {
+                            this.audioSyncAnchor =
+                                this.sharedAudioContext.currentTime - timestamp;
+                            isFirstBuffer = false;
+                        }
+                        this.scheduleAudioChunk(buffer, timestamp);
+                    }
+                    if (timestamp - this.getPlaybackTime() >= 1) {
+                        await new Promise((resolve) => {
+                            const check = () => {
+                                if (timestamp - this.getPlaybackTime() < 1) {
+                                    resolve();
+                                }
+                                else {
+                                    requestAnimationFrame(check);
+                                }
+                            };
+                            check();
+                        });
+                    }
+                }
+            }
+            catch (error) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to run audio iterator', error);
+            }
+        };
+        this.canvas = canvas;
+        this.src = src;
+        this.logLevel = logLevel ?? 'info';
+        this.sharedAudioContext = sharedAudioContext;
+        const context = canvas.getContext('2d', {
+            alpha: false,
+            desynchronized: true,
+        });
+        if (!context) {
+            throw new Error('Could not get 2D context from canvas');
+        }
+        this.context = context;
+    }
+    isReady() {
+        return this.initialized && Boolean(this.sharedAudioContext);
+    }
+    hasAudio() {
+        return Boolean(this.audioSink && this.sharedAudioContext && this.gainNode);
+    }
+    isCurrentlyBuffering() {
+        return this.isBuffering && Boolean(this.bufferingStartedAtMs);
+    }
+    async initialize(startTime = 0) {
+        try {
+            const urlSource = new UrlSource(this.src);
+            const input = new Input({
+                source: urlSource,
+                formats: ALL_FORMATS,
+            });
+            this.input = input;
+            this.totalDuration = await input.computeDuration();
+            const videoTrack = await input.getPrimaryVideoTrack();
+            const audioTrack = await input.getPrimaryAudioTrack();
+            if (!videoTrack && !audioTrack) {
+                throw new Error(`No video or audio track found for ${this.src}`);
+            }
+            if (videoTrack) {
+                this.canvasSink = new CanvasSink(videoTrack, {
+                    poolSize: 2,
+                    fit: 'contain',
+                });
+                this.canvas.width = videoTrack.displayWidth;
+                this.canvas.height = videoTrack.displayHeight;
+            }
+            if (audioTrack && this.sharedAudioContext) {
+                this.audioSink = new AudioBufferSink(audioTrack);
+                this.gainNode = this.sharedAudioContext.createGain();
+                this.gainNode.connect(this.sharedAudioContext.destination);
+            }
+            if (this.sharedAudioContext) {
+                this.audioSyncAnchor = this.sharedAudioContext.currentTime - startTime;
+            }
+            this.initialized = true;
+            await this.startAudioIterator(startTime);
+            await this.startVideoIterator(startTime);
+            this.startRenderLoop();
+        }
+        catch (error) {
+            Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to initialize', error);
+            throw error;
+        }
+    }
+    cleanupAudioQueue() {
+        for (const node of this.queuedAudioNodes) {
+            node.stop();
+        }
+        this.queuedAudioNodes.clear();
+    }
+    async cleanAudioIteratorAndNodes() {
+        await this.audioBufferIterator?.return();
+        this.audioBufferIterator = null;
+        this.audioIteratorStarted = false;
+        this.audioBufferHealth = 0;
+        this.cleanupAudioQueue();
+    }
+    async seekTo(time) {
+        if (!this.isReady())
+            return;
+        const newTime = Math.max(0, Math.min(time, this.totalDuration));
+        const currentPlaybackTime = this.getPlaybackTime();
+        const isSignificantSeek = Math.abs(newTime - currentPlaybackTime) > SEEK_THRESHOLD;
+        if (isSignificantSeek) {
+            this.audioSyncAnchor = this.sharedAudioContext.currentTime - newTime;
+            if (this.audioSink) {
+                await this.cleanAudioIteratorAndNodes();
+            }
+            await this.startAudioIterator(newTime);
+            await this.startVideoIterator(newTime);
+        }
+        if (!this.playing) {
+            this.render();
+        }
+    }
+    async play() {
+        if (!this.isReady())
+            return;
+        if (!this.playing) {
+            if (this.sharedAudioContext.state === 'suspended') {
+                await this.sharedAudioContext.resume();
+            }
+            this.playing = true;
+            this.startRenderLoop();
+        }
+    }
+    pause() {
+        this.playing = false;
+        this.cleanupAudioQueue();
+        this.stopRenderLoop();
+    }
+    dispose() {
+        this.input?.dispose();
+        this.stopRenderLoop();
+        this.videoFrameIterator?.return();
+        this.cleanAudioIteratorAndNodes();
+        this.videoAsyncId++;
+    }
+    getPlaybackTime() {
+        return this.sharedAudioContext.currentTime - this.audioSyncAnchor;
+    }
+    scheduleAudioChunk(buffer, mediaTimestamp) {
+        const targetTime = mediaTimestamp + this.audioSyncAnchor;
+        const delay = targetTime - this.sharedAudioContext.currentTime;
+        const node = this.sharedAudioContext.createBufferSource();
+        node.buffer = buffer;
+        node.connect(this.gainNode);
+        if (delay >= 0) {
+            node.start(targetTime);
+        }
+        else {
+            node.start(this.sharedAudioContext.currentTime, -delay);
+        }
+        this.queuedAudioNodes.add(node);
+        node.onended = () => this.queuedAudioNodes.delete(node);
+    }
+    onBufferingChange(callback) {
+        this.onBufferingChangeCallback = callback;
+    }
+    canRenderVideo() {
+        return (this.audioIteratorStarted &&
+            this.audioBufferHealth >= this.HEALTHY_BUFER_THRESHOLD_SECONDS);
+    }
+    startRenderLoop() {
+        if (this.animationFrameId !== null) {
+            return;
+        }
+        this.render();
+    }
+    stopRenderLoop() {
+        if (this.animationFrameId !== null) {
+            cancelAnimationFrame(this.animationFrameId);
+            this.animationFrameId = null;
+        }
+    }
+    shouldRenderFrame() {
+        return (!this.isBuffering &&
+            this.canRenderVideo() &&
+            this.nextFrame !== null &&
+            this.nextFrame.timestamp <= this.getPlaybackTime());
+    }
+    drawCurrentFrame() {
+        this.context.drawImage(this.nextFrame.canvas, 0, 0);
+        this.nextFrame = null;
+        this.updateNextFrame();
+    }
+    setBufferingState(isBuffering) {
+        if (this.isBuffering !== isBuffering) {
+            this.isBuffering = isBuffering;
+            if (isBuffering) {
+                this.bufferingStartedAtMs = performance.now();
+                this.onBufferingChangeCallback?.(true);
+            }
+            else {
+                this.bufferingStartedAtMs = null;
+                this.onBufferingChangeCallback?.(false);
+            }
+        }
+    }
+    maybeResumeFromBuffering(currentBufferDuration) {
+        if (!this.isCurrentlyBuffering())
+            return;
+        const now = performance.now();
+        const bufferingDuration = now - this.bufferingStartedAtMs;
+        const minTimeElapsed = bufferingDuration >= this.minBufferingTimeoutMs;
+        const bufferHealthy = currentBufferDuration >= this.HEALTHY_BUFER_THRESHOLD_SECONDS;
+        if (minTimeElapsed && bufferHealthy) {
+            Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Resuming from buffering after ${bufferingDuration}ms - buffer recovered`);
+            this.setBufferingState(false);
+        }
+    }
+    maybeForceResumeFromBuffering() {
+        if (!this.isCurrentlyBuffering())
+            return;
+        const now = performance.now();
+        const bufferingDuration = now - this.bufferingStartedAtMs;
+        const forceTimeout = bufferingDuration > this.minBufferingTimeoutMs * 10;
+        if (forceTimeout) {
+            Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Force resuming from buffering after ${bufferingDuration}ms`);
+            this.setBufferingState(false);
+        }
+    }
+}

package/dist/video/new-video-for-preview.d.ts

@@ -0,0 +1,10 @@
+import React from 'react';
+import type { LogLevel } from 'remotion';
+type NewVideoForPreviewProps = {
+    readonly src: string;
+    readonly style?: React.CSSProperties;
+    readonly playbackRate?: number;
+    readonly logLevel?: LogLevel;
+};
+export declare const NewVideoForPreview: React.FC<NewVideoForPreviewProps>;
+export {};

package/dist/video/new-video-for-preview.js

@@ -0,0 +1,108 @@
+import { jsx as _jsx } from "react/jsx-runtime";
+import { useContext, useEffect, useRef, useState } from 'react';
+import { Internals, useBufferState, useCurrentFrame } from 'remotion';
+import { MediaPlayer } from './media-player';
+const { useUnsafeVideoConfig, Timeline, SharedAudioContext } = Internals;
+export const NewVideoForPreview = ({ src, style, playbackRate = 1, logLevel = 'info', }) => {
+    const canvasRef = useRef(null);
+    const videoConfig = useUnsafeVideoConfig();
+    const frame = useCurrentFrame();
+    const mediaPlayerRef = useRef(null);
+    const [mediaPlayerReady, setMediaPlayerReady] = useState(false);
+    const [playing] = Timeline.usePlayingState();
+    const sharedAudioContext = useContext(SharedAudioContext);
+    const buffer = useBufferState();
+    const delayHandleRef = useRef(null);
+    if (!videoConfig) {
+        throw new Error('No video config found');
+    }
+    if (!src) {
+        throw new TypeError('No `src` was passed to <NewVideoForPreview>.');
+    }
+    const actualFps = videoConfig.fps / playbackRate;
+    const currentTime = frame / actualFps;
+    const [initialTimestamp] = useState(currentTime);
+    useEffect(() => {
+        if (!canvasRef.current)
+            return;
+        if (!sharedAudioContext)
+            return;
+        if (!sharedAudioContext.audioContext)
+            return;
+        try {
+            const player = new MediaPlayer({
+                canvas: canvasRef.current,
+                src,
+                logLevel,
+                sharedAudioContext: sharedAudioContext.audioContext,
+            });
+            mediaPlayerRef.current = player;
+            player
+                .initialize(initialTimestamp)
+                .then(() => {
+                setMediaPlayerReady(true);
+                Internals.Log.trace({ logLevel, tag: '@remotion/media' }, `[NewVideoForPreview] MediaPlayer initialized successfully`);
+            })
+                .catch((error) => {
+                Internals.Log.error({ logLevel, tag: '@remotion/media' }, '[NewVideoForPreview] Failed to initialize MediaPlayer', error);
+            });
+        }
+        catch (error) {
+            Internals.Log.error({ logLevel, tag: '@remotion/media' }, '[NewVideoForPreview] MediaPlayer initialization failed', error);
+        }
+        return () => {
+            if (delayHandleRef.current) {
+                delayHandleRef.current.unblock();
+                delayHandleRef.current = null;
+            }
+            if (mediaPlayerRef.current) {
+                Internals.Log.trace({ logLevel, tag: '@remotion/media' }, `[NewVideoForPreview] Disposing MediaPlayer`);
+                mediaPlayerRef.current.dispose();
+                mediaPlayerRef.current = null;
+            }
+            setMediaPlayerReady(false);
+        };
+    }, [src, logLevel, sharedAudioContext, initialTimestamp]);
+    // sync play/pause state with Remotion timeline (like old VideoForPreview video does)
+    useEffect(() => {
+        const mediaPlayer = mediaPlayerRef.current;
+        if (!mediaPlayer)
+            return;
+        if (playing) {
+            mediaPlayer.play().catch((error) => {
+                Internals.Log.error({ logLevel, tag: '@remotion/media' }, '[NewVideoForPreview] Failed to play', error);
+            });
+        }
+        else {
+            mediaPlayer.pause();
+        }
+    }, [playing, logLevel, mediaPlayerReady]);
+    // sync target time with MediaPlayer
+    useEffect(() => {
+        const mediaPlayer = mediaPlayerRef.current;
+        if (!mediaPlayer || !mediaPlayerReady)
+            return;
+        mediaPlayer.seekTo(currentTime);
+        Internals.Log.trace({ logLevel, tag: '@remotion/media' }, `[NewVideoForPreview] Updating target time to ${currentTime.toFixed(3)}s`);
+    }, [currentTime, logLevel, mediaPlayerReady]);
+    // sync MediaPlayer buffering with Remotion buffering
+    useEffect(() => {
+        const mediaPlayer = mediaPlayerRef.current;
+        if (!mediaPlayer || !mediaPlayerReady)
+            return;
+        mediaPlayer.onBufferingChange((newBufferingState) => {
+            if (newBufferingState && !delayHandleRef.current) {
+                // Start blocking Remotion playback
+                delayHandleRef.current = buffer.delayPlayback();
+                Internals.Log.trace({ logLevel, tag: '@remotion/media' }, '[NewVideoForPreview] MediaPlayer buffering - blocking Remotion playback');
+            }
+            else if (!newBufferingState && delayHandleRef.current) {
+                // Unblock Remotion playback
+                delayHandleRef.current.unblock();
+                delayHandleRef.current = null;
+                Internals.Log.trace({ logLevel, tag: '@remotion/media' }, '[NewVideoForPreview] MediaPlayer unbuffering - unblocking Remotion playback');
+            }
+        });
+    }, [mediaPlayerReady, buffer, logLevel]);
+    return (_jsx("canvas", { ref: canvasRef, width: videoConfig.width, height: videoConfig.height, style: style }));
+};

package/dist/video/timeout-utils.d.ts

@@ -0,0 +1,2 @@
+export declare const sleep: (ms: number) => Promise<unknown>;
+export declare function withTimeout<T>(promise: Promise<T>, timeoutMs: number, errorMessage?: string): Promise<T>;

package/dist/video/timeout-utils.js

@@ -0,0 +1,18 @@
+/* eslint-disable no-promise-executor-return */
+export const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+export function withTimeout(promise, timeoutMs, errorMessage = 'Operation timed out') {
+    let timeoutId = null;
+    const timeoutPromise = new Promise((_, reject) => {
+        timeoutId = window.setTimeout(() => {
+            reject(new Error(errorMessage));
+        }, timeoutMs);
+    });
+    return Promise.race([
+        promise.finally(() => {
+            if (timeoutId) {
+                clearTimeout(timeoutId);
+            }
+        }),
+        timeoutPromise,
+    ]);
+}

package/dist/video/video-for-preview.js

@@ -71,6 +71,7 @@ const VideoForPreviewAssertedShowing = ({ src: unpreloadedSrc, style, playbackRa
         throw new Error('useMediaPlayback must be used inside a <BufferingContext>');
     }
     const isPlayerBuffering = Internals.useIsPlayerBuffering(buffering);
+    const initialPlaying = useRef(playing && !isPlayerBuffering);
     const initialIsPremounting = useRef(isPremounting);
     const initialIsPostmounting = useRef(isPostmounting);
     const initialGlobalPlaybackRate = useRef(globalPlaybackRate);
@@ -98,6 +99,7 @@ const VideoForPreviewAssertedShowing = ({ src: unpreloadedSrc, style, playbackRa
                 isPostmounting: initialIsPostmounting.current,
                 globalPlaybackRate: initialGlobalPlaybackRate.current,
                 onVideoFrameCallback: initialOnVideoFrameRef.current ?? null,
+                playing: initialPlaying.current,
             });
             mediaPlayerRef.current = player;
             player

package/dist/video/video-preview-iterator.d.ts

@@ -1,5 +1,6 @@
-import type { CanvasSink, WrappedCanvas } from 'mediabunny';
-export declare const createVideoIterator: (timeToSeek: number, videoSink: CanvasSink) => {
+import type { WrappedCanvas } from 'mediabunny';
+import type { PrewarmedVideoIteratorCache } from '../prewarm-iterator-for-looping';
+export declare const createVideoIterator: (timeToSeek: number, cache: PrewarmedVideoIteratorCache) => {
     destroy: () => void;
     getNext: () => Promise<IteratorResult<WrappedCanvas, void>>;
     isDestroyed: () => boolean;

package/dist/video/video-preview-iterator.js

@@ -1,7 +1,7 @@
 import { roundTo4Digits } from '../helpers/round-to-4-digits';
-export const createVideoIterator = (timeToSeek, videoSink) => {
+export const createVideoIterator = (timeToSeek, cache) => {
     let destroyed = false;
-    const iterator = videoSink.canvases(timeToSeek);
+    const iterator = cache.makeIteratorOrUsePrewarmed(timeToSeek);
     let lastReturnedFrame = null;
     let iteratorEnded = false;
     const getNextOrNullIfNotAvailable = async () => {

package/dist/video-extraction/media-player.d.ts

@@ -0,0 +1,64 @@
+import { type LogLevel } from '../log';
+export declare class MediaPlayer {
+    private canvas;
+    private context;
+    private src;
+    private logLevel;
+    private canvasSink;
+    private videoFrameIterator;
+    private nextFrame;
+    private audioSink;
+    private audioBufferIterator;
+    private queuedAudioNodes;
+    private gainNode;
+    private expectedAudioTime;
+    private sharedAudioContext;
+    private mediaTimeOffset;
+    private playing;
+    private animationFrameId;
+    private asyncId;
+    private initialized;
+    private totalDuration;
+    private actualFps;
+    private isStalled;
+    private onStalledChangeCallback?;
+    private lastAudioProgressAtMs;
+    private lastNetworkActivityAtMs;
+    private isNetworkActive;
+    private isSeeking;
+    private canStartAudio;
+    constructor({ canvas, src, logLevel, sharedAudioContext, }: {
+        canvas: HTMLCanvasElement;
+        src: string;
+        logLevel: LogLevel;
+        sharedAudioContext?: AudioContext | null;
+    });
+    initialize(startTime?: number): Promise<void>;
+    seekTo(time: number): void;
+    drawInitialFrame(time?: number): Promise<void>;
+    play(): Promise<void>;
+    pause(): void;
+    dispose(): void;
+    get currentTime(): number;
+    private getPlaybackTime;
+    get duration(): number;
+    get isPlaying(): boolean;
+    get stalled(): boolean;
+    onStalledChange(callback: (isStalled: boolean) => void): void;
+    private renderSingleFrame;
+    private startRenderLoop;
+    private stopRenderLoop;
+    private render;
+    private startVideoIterator;
+    private updateNextFrame;
+    private tryStartAudio;
+    private getCurrentTimeMs;
+    private resetAudioProgressStopwatch;
+    private getAudioLookaheadSec;
+    private calculateAudioStallThresholdSec;
+    private isNetworkStalled;
+    private checkVideoStall;
+    private checkIfStalled;
+    private updateStalledState;
+    private runAudioIterator;
+}