@remotion/media 4.0.363 → 4.0.364

package/dist/helpers/round-to-4-digits.d.ts

@@ -0,0 +1 @@
+export declare const roundTo4Digits: (timestamp: number) => number;

package/dist/helpers/round-to-4-digits.js

@@ -0,0 +1,4 @@
+// Round to only 4 digits, because WebM has a timescale of 1_000, e.g. framer.webm
+export const roundTo4Digits = (timestamp) => {
+    return Math.round(timestamp * 1000) / 1000;
+};

package/dist/media-player.d.ts

@@ -0,0 +1,95 @@
+import type { LogLevel, useBufferState } from 'remotion';
+export type MediaPlayerInitResult = {
+    type: 'success';
+    durationInSeconds: number;
+} | {
+    type: 'unknown-container-format';
+} | {
+    type: 'cannot-decode';
+} | {
+    type: 'network-error';
+} | {
+    type: 'no-tracks';
+} | {
+    type: 'disposed';
+};
+export declare class MediaPlayer {
+    private canvas;
+    private context;
+    private src;
+    private logLevel;
+    private playbackRate;
+    private audioStreamIndex;
+    private canvasSink;
+    private videoFrameIterator;
+    private debugStats;
+    private audioSink;
+    private audioBufferIterator;
+    private gainNode;
+    private currentVolume;
+    private sharedAudioContext;
+    private audioSyncAnchor;
+    private playing;
+    private muted;
+    private loop;
+    private fps;
+    private trimBefore;
+    private trimAfter;
+    private initialized;
+    private totalDuration;
+    private isBuffering;
+    private onBufferingChangeCallback?;
+    private mediaEnded;
+    private debugOverlay;
+    private onVideoFrameCallback?;
+    private initializationPromise;
+    private bufferState;
+    constructor({ canvas, src, logLevel, sharedAudioContext, loop, trimBefore, trimAfter, playbackRate, audioStreamIndex, fps, debugOverlay, bufferState, }: {
+        canvas: HTMLCanvasElement | null;
+        src: string;
+        logLevel: LogLevel;
+        sharedAudioContext: AudioContext;
+        loop: boolean;
+        trimBefore: number | undefined;
+        trimAfter: number | undefined;
+        playbackRate: number;
+        audioStreamIndex: number;
+        fps: number;
+        debugOverlay: boolean;
+        bufferState: ReturnType<typeof useBufferState>;
+    });
+    private input;
+    private isReady;
+    private hasAudio;
+    private isCurrentlyBuffering;
+    private isDisposalError;
+    initialize(startTimeUnresolved: number): Promise<MediaPlayerInitResult>;
+    private _initialize;
+    private clearCanvas;
+    private currentSeekNonce;
+    private seekPromiseChain;
+    seekTo(time: number): Promise<void>;
+    seekToDoNotCallDirectly(time: number, nonce: number): Promise<void>;
+    play(): Promise<void>;
+    pause(): void;
+    setMuted(muted: boolean): void;
+    setVolume(volume: number): void;
+    setDebugOverlay(debugOverlay: boolean): void;
+    setPlaybackRate(rate: number): void;
+    setFps(fps: number): void;
+    setLoop(loop: boolean): void;
+    dispose(): Promise<void>;
+    private getPlaybackTime;
+    private scheduleAudioChunk;
+    onBufferingChange(callback: (isBuffering: boolean) => void): () => void;
+    onVideoFrame(callback: (frame: CanvasImageSource) => void): () => void;
+    private drawFrame;
+    private startAudioIterator;
+    private drawDebugOverlay;
+    private startVideoIterator;
+    private bufferingStartedAtMs;
+    private minBufferingTimeoutMs;
+    private setBufferingState;
+    private maybeResumeFromBuffering;
+    private runAudioIterator;
+}

package/dist/media-player.js

@@ -0,0 +1,496 @@
+import { ALL_FORMATS, AudioBufferSink, CanvasSink, Input, UrlSource, } from 'mediabunny';
+import { Internals } from 'remotion';
+import { HEALTHY_BUFFER_THRESHOLD_SECONDS, makeAudioIterator, } from './audio/audio-preview-iterator';
+import { drawPreviewOverlay } from './debug-overlay/preview-overlay';
+import { getTimeInSeconds } from './get-time-in-seconds';
+import { isNetworkError } from './is-network-error';
+import { sleep, TimeoutError, withTimeout } from './video/timeout-utils';
+import { createVideoIterator, } from './video/video-preview-iterator';
+const AUDIO_BUFFER_TOLERANCE_THRESHOLD = 0.1;
+export class MediaPlayer {
+    constructor({ canvas, src, logLevel, sharedAudioContext, loop, trimBefore, trimAfter, playbackRate, audioStreamIndex, fps, debugOverlay, bufferState, }) {
+        this.canvasSink = null;
+        this.videoFrameIterator = null;
+        this.debugStats = {
+            videoIteratorsCreated: 0,
+            framesRendered: 0,
+        };
+        this.audioSink = null;
+        this.audioBufferIterator = null;
+        this.gainNode = null;
+        this.currentVolume = 1;
+        // this is the time difference between Web Audio timeline
+        // and media file timeline
+        this.audioSyncAnchor = 0;
+        this.playing = false;
+        this.muted = false;
+        this.loop = false;
+        this.initialized = false;
+        // for remotion buffer state
+        this.isBuffering = false;
+        this.mediaEnded = false;
+        this.debugOverlay = false;
+        this.initializationPromise = null;
+        this.input = null;
+        this.currentSeekNonce = 0;
+        this.seekPromiseChain = Promise.resolve();
+        this.drawFrame = (frame) => {
+            if (!this.context) {
+                throw new Error('Context not initialized');
+            }
+            this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
+            this.context.drawImage(frame.canvas, 0, 0);
+            this.debugStats.framesRendered++;
+            this.drawDebugOverlay();
+            if (this.onVideoFrameCallback && this.canvas) {
+                this.onVideoFrameCallback(this.canvas);
+            }
+            Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Drew frame ${frame.timestamp.toFixed(3)}s`);
+        };
+        this.startAudioIterator = (startFromSecond) => {
+            if (!this.hasAudio())
+                return;
+            // Clean up existing audio iterator
+            this.audioBufferIterator?.destroy();
+            try {
+                const iterator = makeAudioIterator(this.audioSink, startFromSecond);
+                this.audioBufferIterator = iterator;
+                this.runAudioIterator(startFromSecond, iterator);
+            }
+            catch (error) {
+                if (this.isDisposalError()) {
+                    return;
+                }
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start audio iterator', error);
+            }
+        };
+        this.startVideoIterator = async (timeToSeek, nonce) => {
+            if (!this.canvasSink) {
+                return;
+            }
+            this.videoFrameIterator?.destroy();
+            const iterator = createVideoIterator(timeToSeek, this.canvasSink);
+            this.debugStats.videoIteratorsCreated++;
+            this.videoFrameIterator = iterator;
+            const delayHandle = this.bufferState?.delayPlayback();
+            const frameResult = await iterator.getNext();
+            delayHandle?.unblock();
+            if (iterator.isDestroyed()) {
+                return;
+            }
+            if (nonce !== this.currentSeekNonce) {
+                return;
+            }
+            if (this.videoFrameIterator.isDestroyed()) {
+                return;
+            }
+            if (frameResult.value) {
+                this.audioSyncAnchor =
+                    this.sharedAudioContext.currentTime - frameResult.value.timestamp;
+                this.drawFrame(frameResult.value);
+            }
+            else {
+                // media ended
+            }
+        };
+        this.bufferingStartedAtMs = null;
+        this.minBufferingTimeoutMs = 500;
+        this.runAudioIterator = async (startFromSecond, audioIterator) => {
+            if (!this.hasAudio())
+                return;
+            try {
+                let totalBufferDuration = 0;
+                let isFirstBuffer = true;
+                audioIterator.setAudioIteratorStarted(true);
+                while (true) {
+                    if (audioIterator.isDestroyed()) {
+                        return;
+                    }
+                    const BUFFERING_TIMEOUT_MS = 50;
+                    let result;
+                    try {
+                        result = await withTimeout(audioIterator.getNext(), BUFFERING_TIMEOUT_MS, 'Iterator timeout');
+                    }
+                    catch (error) {
+                        if (error instanceof TimeoutError && !this.mediaEnded) {
+                            this.setBufferingState(true);
+                        }
+                        await sleep(10);
+                        continue;
+                    }
+                    // media has ended
+                    if (result.done || !result.value) {
+                        this.mediaEnded = true;
+                        break;
+                    }
+                    const { buffer, timestamp, duration } = result.value;
+                    totalBufferDuration += duration;
+                    audioIterator.setAudioBufferHealth(Math.max(0, totalBufferDuration / this.playbackRate));
+                    this.maybeResumeFromBuffering(totalBufferDuration / this.playbackRate);
+                    if (this.playing) {
+                        if (isFirstBuffer) {
+                            this.audioSyncAnchor =
+                                this.sharedAudioContext.currentTime - timestamp;
+                            isFirstBuffer = false;
+                        }
+                        // if timestamp is less than timeToSeek, skip
+                        // context: for some reason, mediabunny returns buffer at 9.984s, when requested at 10s
+                        if (timestamp < startFromSecond - AUDIO_BUFFER_TOLERANCE_THRESHOLD) {
+                            continue;
+                        }
+                        this.scheduleAudioChunk(buffer, timestamp);
+                    }
+                    const playbackTime = this.getPlaybackTime();
+                    if (playbackTime === null) {
+                        continue;
+                    }
+                    if (timestamp - playbackTime >= 1) {
+                        await new Promise((resolve) => {
+                            const check = () => {
+                                const currentPlaybackTime = this.getPlaybackTime();
+                                if (currentPlaybackTime !== null &&
+                                    timestamp - currentPlaybackTime < 1) {
+                                    resolve();
+                                }
+                                else {
+                                    requestAnimationFrame(check);
+                                }
+                            };
+                            check();
+                        });
+                    }
+                }
+            }
+            catch (error) {
+                if (this.isDisposalError()) {
+                    return;
+                }
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to run audio iterator', error);
+            }
+        };
+        this.canvas = canvas ?? null;
+        this.src = src;
+        this.logLevel = logLevel ?? window.remotion_logLevel;
+        this.sharedAudioContext = sharedAudioContext;
+        this.playbackRate = playbackRate;
+        this.loop = loop;
+        this.trimBefore = trimBefore;
+        this.trimAfter = trimAfter;
+        this.audioStreamIndex = audioStreamIndex ?? 0;
+        this.fps = fps;
+        this.debugOverlay = debugOverlay;
+        this.bufferState = bufferState;
+        if (canvas) {
+            const context = canvas.getContext('2d', {
+                alpha: true,
+                desynchronized: true,
+            });
+            if (!context) {
+                throw new Error('Could not get 2D context from canvas');
+            }
+            this.context = context;
+        }
+        else {
+            this.context = null;
+        }
+    }
+    isReady() {
+        return (this.initialized &&
+            Boolean(this.sharedAudioContext) &&
+            !this.input?.disposed);
+    }
+    hasAudio() {
+        return Boolean(this.audioSink && this.sharedAudioContext && this.gainNode);
+    }
+    isCurrentlyBuffering() {
+        return this.isBuffering && Boolean(this.bufferingStartedAtMs);
+    }
+    isDisposalError() {
+        return this.input?.disposed === true;
+    }
+    initialize(startTimeUnresolved) {
+        const promise = this._initialize(startTimeUnresolved);
+        this.initializationPromise = promise;
+        return promise;
+    }
+    async _initialize(startTimeUnresolved) {
+        try {
+            const urlSource = new UrlSource(this.src);
+            const input = new Input({
+                source: urlSource,
+                formats: ALL_FORMATS,
+            });
+            this.input = input;
+            if (input.disposed) {
+                return { type: 'disposed' };
+            }
+            try {
+                await input.getFormat();
+            }
+            catch (error) {
+                if (this.isDisposalError()) {
+                    return { type: 'disposed' };
+                }
+                const err = error;
+                if (isNetworkError(err)) {
+                    throw error;
+                }
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Failed to recognize format for ${this.src}`, error);
+                return { type: 'unknown-container-format' };
+            }
+            const [durationInSeconds, videoTrack, audioTracks] = await Promise.all([
+                input.computeDuration(),
+                input.getPrimaryVideoTrack(),
+                input.getAudioTracks(),
+            ]);
+            this.totalDuration = durationInSeconds;
+            const audioTrack = audioTracks[this.audioStreamIndex] ?? null;
+            if (!videoTrack && !audioTrack) {
+                return { type: 'no-tracks' };
+            }
+            if (videoTrack && this.canvas && this.context) {
+                const canDecode = await videoTrack.canDecode();
+                if (!canDecode) {
+                    return { type: 'cannot-decode' };
+                }
+                this.canvasSink = new CanvasSink(videoTrack, {
+                    poolSize: 2,
+                    fit: 'contain',
+                    alpha: true,
+                });
+                this.canvas.width = videoTrack.displayWidth;
+                this.canvas.height = videoTrack.displayHeight;
+            }
+            if (audioTrack && this.sharedAudioContext) {
+                this.audioSink = new AudioBufferSink(audioTrack);
+                this.gainNode = this.sharedAudioContext.createGain();
+                this.gainNode.connect(this.sharedAudioContext.destination);
+            }
+            const startTime = getTimeInSeconds({
+                unloopedTimeInSeconds: startTimeUnresolved,
+                playbackRate: this.playbackRate,
+                loop: this.loop,
+                trimBefore: this.trimBefore,
+                trimAfter: this.trimAfter,
+                mediaDurationInSeconds: this.totalDuration,
+                fps: this.fps,
+                ifNoMediaDuration: 'infinity',
+                src: this.src,
+            });
+            if (startTime === null) {
+                this.clearCanvas();
+                return { type: 'success', durationInSeconds: this.totalDuration };
+            }
+            if (this.sharedAudioContext) {
+                this.audioSyncAnchor = this.sharedAudioContext.currentTime - startTime;
+            }
+            this.initialized = true;
+            try {
+                this.startAudioIterator(startTime);
+                await this.startVideoIterator(startTime, this.currentSeekNonce);
+            }
+            catch (error) {
+                if (this.isDisposalError()) {
+                    return { type: 'disposed' };
+                }
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start audio and video iterators', error);
+            }
+            return { type: 'success', durationInSeconds };
+        }
+        catch (error) {
+            const err = error;
+            if (isNetworkError(err)) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Network/CORS error for ${this.src}`, err);
+                return { type: 'network-error' };
+            }
+            Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to initialize', error);
+            throw error;
+        }
+    }
+    clearCanvas() {
+        if (this.context && this.canvas) {
+            this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
+        }
+    }
+    async seekTo(time) {
+        this.currentSeekNonce++;
+        const nonce = this.currentSeekNonce;
+        await this.seekPromiseChain;
+        this.seekPromiseChain = this.seekToDoNotCallDirectly(time, nonce);
+        await this.seekPromiseChain;
+    }
+    async seekToDoNotCallDirectly(time, nonce) {
+        if (nonce !== this.currentSeekNonce) {
+            return;
+        }
+        if (!this.isReady())
+            return;
+        const newTime = getTimeInSeconds({
+            unloopedTimeInSeconds: time,
+            playbackRate: this.playbackRate,
+            loop: this.loop,
+            trimBefore: this.trimBefore,
+            trimAfter: this.trimAfter,
+            mediaDurationInSeconds: this.totalDuration ?? null,
+            fps: this.fps,
+            ifNoMediaDuration: 'infinity',
+            src: this.src,
+        });
+        if (newTime === null) {
+            // invalidate in-flight video operations
+            this.videoFrameIterator?.destroy();
+            this.videoFrameIterator = null;
+            this.clearCanvas();
+            this.audioBufferIterator?.destroy();
+            this.audioBufferIterator = null;
+            return;
+        }
+        const currentPlaybackTime = this.getPlaybackTime();
+        if (currentPlaybackTime === newTime) {
+            return;
+        }
+        const satisfyResult = await this.videoFrameIterator?.tryToSatisfySeek(newTime);
+        if (satisfyResult?.type === 'satisfied') {
+            this.drawFrame(satisfyResult.frame);
+            return;
+        }
+        if (this.currentSeekNonce !== nonce) {
+            return;
+        }
+        this.mediaEnded = false;
+        this.audioSyncAnchor = this.sharedAudioContext.currentTime - newTime;
+        this.startAudioIterator(newTime);
+        this.startVideoIterator(newTime, nonce);
+    }
+    async play() {
+        if (!this.isReady())
+            return;
+        if (!this.playing) {
+            if (this.sharedAudioContext.state === 'suspended') {
+                await this.sharedAudioContext.resume();
+            }
+            this.playing = true;
+        }
+    }
+    pause() {
+        this.playing = false;
+        this.audioBufferIterator?.cleanupAudioQueue();
+    }
+    setMuted(muted) {
+        this.muted = muted;
+        if (this.gainNode) {
+            this.gainNode.gain.value = muted ? 0 : this.currentVolume;
+        }
+    }
+    setVolume(volume) {
+        if (!this.gainNode) {
+            return;
+        }
+        const appliedVolume = Math.max(0, volume);
+        this.currentVolume = appliedVolume;
+        if (!this.muted) {
+            this.gainNode.gain.value = appliedVolume;
+        }
+    }
+    setDebugOverlay(debugOverlay) {
+        this.debugOverlay = debugOverlay;
+    }
+    setPlaybackRate(rate) {
+        this.playbackRate = rate;
+    }
+    setFps(fps) {
+        this.fps = fps;
+    }
+    setLoop(loop) {
+        this.loop = loop;
+    }
+    async dispose() {
+        this.initialized = false;
+        if (this.initializationPromise) {
+            try {
+                // wait for the init to finished
+                // otherwise we might get errors like:
+                // Error: Response stream reader stopped unexpectedly before all requested data was read. from UrlSource
+                await this.initializationPromise;
+            }
+            catch {
+                // Ignore initialization errors during disposal
+            }
+        }
+        this.input?.dispose();
+        this.videoFrameIterator?.destroy();
+        this.videoFrameIterator = null;
+        this.audioBufferIterator?.destroy();
+        this.audioBufferIterator = null;
+    }
+    getPlaybackTime() {
+        return this.sharedAudioContext.currentTime - this.audioSyncAnchor;
+    }
+    scheduleAudioChunk(buffer, mediaTimestamp) {
+        const targetTime = mediaTimestamp + this.audioSyncAnchor;
+        const delay = targetTime - this.sharedAudioContext.currentTime;
+        const node = this.sharedAudioContext.createBufferSource();
+        node.buffer = buffer;
+        node.playbackRate.value = this.playbackRate;
+        node.connect(this.gainNode);
+        if (delay >= 0) {
+            node.start(targetTime);
+        }
+        else {
+            node.start(this.sharedAudioContext.currentTime, -delay);
+        }
+        this.audioBufferIterator?.addQueuedAudioNode(node);
+        node.onended = () => this.audioBufferIterator?.removeQueuedAudioNode(node);
+    }
+    onBufferingChange(callback) {
+        this.onBufferingChangeCallback = callback;
+        return () => {
+            if (this.onBufferingChangeCallback === callback) {
+                this.onBufferingChangeCallback = undefined;
+            }
+        };
+    }
+    onVideoFrame(callback) {
+        this.onVideoFrameCallback = callback;
+        if (this.initialized && callback && this.canvas) {
+            callback(this.canvas);
+        }
+        return () => {
+            if (this.onVideoFrameCallback === callback) {
+                this.onVideoFrameCallback = undefined;
+            }
+        };
+    }
+    drawDebugOverlay() {
+        if (!this.debugOverlay)
+            return;
+        if (this.context && this.canvas) {
+            drawPreviewOverlay(this.context, this.debugStats, this.sharedAudioContext.state, this.sharedAudioContext.currentTime);
+        }
+    }
+    setBufferingState(isBuffering) {
+        if (this.isBuffering !== isBuffering) {
+            this.isBuffering = isBuffering;
+            if (isBuffering) {
+                this.bufferingStartedAtMs = performance.now();
+                this.onBufferingChangeCallback?.(true);
+            }
+            else {
+                this.bufferingStartedAtMs = null;
+                this.onBufferingChangeCallback?.(false);
+            }
+        }
+    }
+    maybeResumeFromBuffering(currentBufferDuration) {
+        if (!this.isCurrentlyBuffering())
+            return;
+        const now = performance.now();
+        const bufferingDuration = now - this.bufferingStartedAtMs;
+        const minTimeElapsed = bufferingDuration >= this.minBufferingTimeoutMs;
+        const bufferHealthy = currentBufferDuration >= HEALTHY_BUFFER_THRESHOLD_SECONDS;
+        if (minTimeElapsed && bufferHealthy) {
+            Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Resuming from buffering after ${bufferingDuration}ms - buffer recovered`);
+            this.setBufferingState(false);
+        }
+    }
+}

package/dist/video/props.d.ts

@@ -37,6 +37,7 @@ type OptionalVideoProps = {
     trimBefore: number | undefined;
     toneFrequency: number;
     showInTimeline: boolean;
+    debugOverlay: boolean;
 };
 export type InnerVideoProps = MandatoryVideoProps & OuterVideoProps & OptionalVideoProps;
 export type VideoProps = MandatoryVideoProps & Partial<OuterVideoProps> & Partial<OptionalVideoProps>;

package/dist/video/video-for-preview.d.ts

@@ -1,7 +1,7 @@
 import React from 'react';
 import type { LogLevel, LoopVolumeCurveBehavior, VolumeProp } from 'remotion';
 import type { FallbackOffthreadVideoProps } from './props';
-type NewVideoForPreviewProps = {
+type VideoForPreviewProps = {
     readonly src: string;
     readonly style: React.CSSProperties | undefined;
     readonly playbackRate: number;
@@ -20,6 +20,7 @@ type NewVideoForPreviewProps = {
     readonly disallowFallbackToOffthreadVideo: boolean;
     readonly fallbackOffthreadVideoProps: FallbackOffthreadVideoProps;
     readonly audioStreamIndex: number;
+    readonly debugOverlay: boolean;
 };
-export declare const VideoForPreview: React.FC<NewVideoForPreviewProps>;
+export declare const VideoForPreview: React.FC<VideoForPreviewProps>;
 export {};