@remotion/media 4.0.363 → 4.0.365

package/dist/helpers/round-to-4-digits.d.ts

@@ -0,0 +1 @@
+export declare const roundTo4Digits: (timestamp: number) => number;

package/dist/helpers/round-to-4-digits.js

@@ -0,0 +1,4 @@
+// Round to only 4 digits, because WebM has a timescale of 1_000, e.g. framer.webm
+export const roundTo4Digits = (timestamp) => {
+    return Math.round(timestamp * 1000) / 1000;
+};

package/dist/media-player.d.ts

@@ -0,0 +1,87 @@
+import type { LogLevel, useBufferState } from 'remotion';
+export type MediaPlayerInitResult = {
+    type: 'success';
+    durationInSeconds: number;
+} | {
+    type: 'unknown-container-format';
+} | {
+    type: 'cannot-decode';
+} | {
+    type: 'network-error';
+} | {
+    type: 'no-tracks';
+} | {
+    type: 'disposed';
+};
+export declare class MediaPlayer {
+    private canvas;
+    private context;
+    private src;
+    private logLevel;
+    private playbackRate;
+    private audioStreamIndex;
+    private canvasSink;
+    private videoFrameIterator;
+    private debugStats;
+    private audioSink;
+    private audioBufferIterator;
+    private gainNode;
+    private currentVolume;
+    private sharedAudioContext;
+    private audioSyncAnchor;
+    private playing;
+    private muted;
+    private loop;
+    private fps;
+    private trimBefore;
+    private trimAfter;
+    private initialized;
+    private totalDuration;
+    private debugOverlay;
+    private onVideoFrameCallback?;
+    private initializationPromise;
+    private bufferState;
+    constructor({ canvas, src, logLevel, sharedAudioContext, loop, trimBefore, trimAfter, playbackRate, audioStreamIndex, fps, debugOverlay, bufferState, }: {
+        canvas: HTMLCanvasElement | null;
+        src: string;
+        logLevel: LogLevel;
+        sharedAudioContext: AudioContext;
+        loop: boolean;
+        trimBefore: number | undefined;
+        trimAfter: number | undefined;
+        playbackRate: number;
+        audioStreamIndex: number;
+        fps: number;
+        debugOverlay: boolean;
+        bufferState: ReturnType<typeof useBufferState>;
+    });
+    private input;
+    private isReady;
+    private hasAudio;
+    private isDisposalError;
+    initialize(startTimeUnresolved: number): Promise<MediaPlayerInitResult>;
+    private _initialize;
+    private clearCanvas;
+    private currentSeekNonce;
+    private seekPromiseChain;
+    seekTo(time: number): Promise<void>;
+    seekToDoNotCallDirectly(time: number, nonce: number): Promise<void>;
+    play(time: number): Promise<void>;
+    pause(): void;
+    setMuted(muted: boolean): void;
+    setVolume(volume: number): void;
+    setDebugOverlay(debugOverlay: boolean): void;
+    setPlaybackRate(rate: number): void;
+    setFps(fps: number): void;
+    setLoop(loop: boolean): void;
+    dispose(): Promise<void>;
+    private getPlaybackTime;
+    private setPlaybackTime;
+    private audioChunksForAfterResuming;
+    private scheduleAudioChunk;
+    onVideoFrame(callback: (frame: CanvasImageSource) => void): () => void;
+    private drawFrame;
+    private startAudioIterator;
+    private drawDebugOverlay;
+    private startVideoIterator;
+}

package/dist/media-player.js

@@ -0,0 +1,475 @@
+import { ALL_FORMATS, AudioBufferSink, CanvasSink, Input, UrlSource, } from 'mediabunny';
+import { Internals } from 'remotion';
+import { isAlreadyQueued, makeAudioIterator, } from './audio/audio-preview-iterator';
+import { drawPreviewOverlay } from './debug-overlay/preview-overlay';
+import { getTimeInSeconds } from './get-time-in-seconds';
+import { isNetworkError } from './is-network-error';
+import { createVideoIterator, } from './video/video-preview-iterator';
+export class MediaPlayer {
+    constructor({ canvas, src, logLevel, sharedAudioContext, loop, trimBefore, trimAfter, playbackRate, audioStreamIndex, fps, debugOverlay, bufferState, }) {
+        this.canvasSink = null;
+        this.videoFrameIterator = null;
+        this.debugStats = {
+            videoIteratorsCreated: 0,
+            audioIteratorsCreated: 0,
+            framesRendered: 0,
+        };
+        this.audioSink = null;
+        this.audioBufferIterator = null;
+        this.gainNode = null;
+        this.currentVolume = 1;
+        // this is the time difference between Web Audio timeline
+        // and media file timeline
+        this.audioSyncAnchor = 0;
+        this.playing = false;
+        this.muted = false;
+        this.loop = false;
+        this.initialized = false;
+        this.debugOverlay = false;
+        this.initializationPromise = null;
+        this.input = null;
+        this.currentSeekNonce = 0;
+        this.seekPromiseChain = Promise.resolve();
+        this.audioChunksForAfterResuming = [];
+        this.drawFrame = (frame) => {
+            if (!this.context) {
+                throw new Error('Context not initialized');
+            }
+            this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
+            this.context.drawImage(frame.canvas, 0, 0);
+            this.debugStats.framesRendered++;
+            this.drawDebugOverlay();
+            if (this.onVideoFrameCallback && this.canvas) {
+                this.onVideoFrameCallback(this.canvas);
+            }
+            Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Drew frame ${frame.timestamp.toFixed(3)}s`);
+        };
+        this.startAudioIterator = async (startFromSecond, nonce) => {
+            if (!this.hasAudio())
+                return;
+            this.audioBufferIterator?.destroy();
+            this.audioChunksForAfterResuming = [];
+            const delayHandle = this.bufferState.delayPlayback();
+            const iterator = makeAudioIterator(this.audioSink, startFromSecond);
+            this.debugStats.audioIteratorsCreated++;
+            this.audioBufferIterator = iterator;
+            // Schedule up to 3 buffers ahead of the current time
+            for (let i = 0; i < 3; i++) {
+                const result = await iterator.getNext();
+                if (iterator.isDestroyed()) {
+                    delayHandle.unblock();
+                    return;
+                }
+                if (nonce !== this.currentSeekNonce) {
+                    delayHandle.unblock();
+                    return;
+                }
+                if (!result.value) {
+                    // media ended
+                    delayHandle.unblock();
+                    return;
+                }
+                const { buffer, timestamp } = result.value;
+                this.audioChunksForAfterResuming.push({
+                    buffer,
+                    timestamp,
+                });
+            }
+            delayHandle.unblock();
+        };
+        this.startVideoIterator = async (timeToSeek, nonce) => {
+            if (!this.canvasSink) {
+                return;
+            }
+            this.videoFrameIterator?.destroy();
+            const iterator = createVideoIterator(timeToSeek, this.canvasSink);
+            this.debugStats.videoIteratorsCreated++;
+            this.videoFrameIterator = iterator;
+            const delayHandle = this.bufferState.delayPlayback();
+            const frameResult = await iterator.getNext();
+            delayHandle.unblock();
+            if (iterator.isDestroyed()) {
+                return;
+            }
+            if (nonce !== this.currentSeekNonce) {
+                return;
+            }
+            if (this.videoFrameIterator.isDestroyed()) {
+                return;
+            }
+            if (!frameResult.value) {
+                // media ended
+                return;
+            }
+            this.drawFrame(frameResult.value);
+        };
+        this.canvas = canvas ?? null;
+        this.src = src;
+        this.logLevel = logLevel ?? window.remotion_logLevel;
+        this.sharedAudioContext = sharedAudioContext;
+        this.playbackRate = playbackRate;
+        this.loop = loop;
+        this.trimBefore = trimBefore;
+        this.trimAfter = trimAfter;
+        this.audioStreamIndex = audioStreamIndex ?? 0;
+        this.fps = fps;
+        this.debugOverlay = debugOverlay;
+        this.bufferState = bufferState;
+        if (canvas) {
+            const context = canvas.getContext('2d', {
+                alpha: true,
+                desynchronized: true,
+            });
+            if (!context) {
+                throw new Error('Could not get 2D context from canvas');
+            }
+            this.context = context;
+        }
+        else {
+            this.context = null;
+        }
+    }
+    isReady() {
+        return (this.initialized &&
+            Boolean(this.sharedAudioContext) &&
+            !this.input?.disposed);
+    }
+    hasAudio() {
+        return Boolean(this.audioSink && this.sharedAudioContext && this.gainNode);
+    }
+    isDisposalError() {
+        return this.input?.disposed === true;
+    }
+    initialize(startTimeUnresolved) {
+        const promise = this._initialize(startTimeUnresolved);
+        this.initializationPromise = promise;
+        return promise;
+    }
+    async _initialize(startTimeUnresolved) {
+        try {
+            const urlSource = new UrlSource(this.src);
+            const input = new Input({
+                source: urlSource,
+                formats: ALL_FORMATS,
+            });
+            this.input = input;
+            if (input.disposed) {
+                return { type: 'disposed' };
+            }
+            try {
+                await input.getFormat();
+            }
+            catch (error) {
+                if (this.isDisposalError()) {
+                    return { type: 'disposed' };
+                }
+                const err = error;
+                if (isNetworkError(err)) {
+                    throw error;
+                }
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Failed to recognize format for ${this.src}`, error);
+                return { type: 'unknown-container-format' };
+            }
+            const [durationInSeconds, videoTrack, audioTracks] = await Promise.all([
+                input.computeDuration(),
+                input.getPrimaryVideoTrack(),
+                input.getAudioTracks(),
+            ]);
+            this.totalDuration = durationInSeconds;
+            const audioTrack = audioTracks[this.audioStreamIndex] ?? null;
+            if (!videoTrack && !audioTrack) {
+                return { type: 'no-tracks' };
+            }
+            if (videoTrack && this.canvas && this.context) {
+                const canDecode = await videoTrack.canDecode();
+                if (!canDecode) {
+                    return { type: 'cannot-decode' };
+                }
+                this.canvasSink = new CanvasSink(videoTrack, {
+                    poolSize: 2,
+                    fit: 'contain',
+                    alpha: true,
+                });
+                this.canvas.width = videoTrack.displayWidth;
+                this.canvas.height = videoTrack.displayHeight;
+            }
+            if (audioTrack && this.sharedAudioContext) {
+                this.audioSink = new AudioBufferSink(audioTrack);
+                this.gainNode = this.sharedAudioContext.createGain();
+                this.gainNode.connect(this.sharedAudioContext.destination);
+            }
+            const startTime = getTimeInSeconds({
+                unloopedTimeInSeconds: startTimeUnresolved,
+                playbackRate: this.playbackRate,
+                loop: this.loop,
+                trimBefore: this.trimBefore,
+                trimAfter: this.trimAfter,
+                mediaDurationInSeconds: this.totalDuration,
+                fps: this.fps,
+                ifNoMediaDuration: 'infinity',
+                src: this.src,
+            });
+            if (startTime === null) {
+                this.clearCanvas();
+                return { type: 'success', durationInSeconds: this.totalDuration };
+            }
+            if (this.sharedAudioContext) {
+                this.setPlaybackTime(startTime);
+            }
+            this.initialized = true;
+            try {
+                // intentionally not awaited
+                this.startAudioIterator(startTime, this.currentSeekNonce);
+                await this.startVideoIterator(startTime, this.currentSeekNonce);
+            }
+            catch (error) {
+                if (this.isDisposalError()) {
+                    return { type: 'disposed' };
+                }
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start audio and video iterators', error);
+            }
+            return { type: 'success', durationInSeconds };
+        }
+        catch (error) {
+            const err = error;
+            if (isNetworkError(err)) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Network/CORS error for ${this.src}`, err);
+                return { type: 'network-error' };
+            }
+            Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to initialize', error);
+            throw error;
+        }
+    }
+    clearCanvas() {
+        if (this.context && this.canvas) {
+            this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
+        }
+    }
+    async seekTo(time) {
+        this.currentSeekNonce++;
+        const nonce = this.currentSeekNonce;
+        await this.seekPromiseChain;
+        this.seekPromiseChain = this.seekToDoNotCallDirectly(time, nonce);
+        await this.seekPromiseChain;
+    }
+    async seekToDoNotCallDirectly(time, nonce) {
+        if (nonce !== this.currentSeekNonce) {
+            return;
+        }
+        if (!this.isReady())
+            return;
+        const newTime = getTimeInSeconds({
+            unloopedTimeInSeconds: time,
+            playbackRate: this.playbackRate,
+            loop: this.loop,
+            trimBefore: this.trimBefore,
+            trimAfter: this.trimAfter,
+            mediaDurationInSeconds: this.totalDuration ?? null,
+            fps: this.fps,
+            ifNoMediaDuration: 'infinity',
+            src: this.src,
+        });
+        if (newTime === null) {
+            // invalidate in-flight video operations
+            this.videoFrameIterator?.destroy();
+            this.videoFrameIterator = null;
+            this.clearCanvas();
+            this.audioBufferIterator?.destroy();
+            this.audioBufferIterator = null;
+            return;
+        }
+        const currentPlaybackTime = this.getPlaybackTime();
+        if (currentPlaybackTime === newTime) {
+            return;
+        }
+        const newAudioSyncAnchor = this.sharedAudioContext.currentTime - newTime;
+        const diff = Math.abs(newAudioSyncAnchor - this.audioSyncAnchor);
+        if (diff > 0.1) {
+            this.setPlaybackTime(newTime);
+        }
+        // Should return immediately, so it's okay to not use Promise.all here
+        const videoSatisfyResult = await this.videoFrameIterator?.tryToSatisfySeek(newTime);
+        if (videoSatisfyResult?.type === 'satisfied') {
+            this.drawFrame(videoSatisfyResult.frame);
+        }
+        else if (videoSatisfyResult && this.currentSeekNonce === nonce) {
+            this.startVideoIterator(newTime, nonce);
+        }
+        const queuedPeriod = this.audioBufferIterator?.getQueuedPeriod();
+        const currentTimeIsAlreadyQueued = isAlreadyQueued(newTime, queuedPeriod);
+        const toBeScheduled = [];
+        if (!currentTimeIsAlreadyQueued) {
+            const audioSatisfyResult = await this.audioBufferIterator?.tryToSatisfySeek(newTime);
+            if (this.currentSeekNonce !== nonce) {
+                return;
+            }
+            if (!audioSatisfyResult) {
+                return;
+            }
+            if (audioSatisfyResult.type === 'not-satisfied') {
+                await this.startAudioIterator(newTime, nonce);
+                return;
+            }
+            toBeScheduled.push(...audioSatisfyResult.buffers);
+        }
+        // TODO: What is this is beyond the end of the video
+        const nextTime = newTime +
+            // start of next frame
+            (1 / this.fps) * this.playbackRate +
+            // need the full duration of the next frame to be queued
+            (1 / this.fps) * this.playbackRate;
+        const nextIsAlreadyQueued = isAlreadyQueued(nextTime, queuedPeriod);
+        if (!nextIsAlreadyQueued) {
+            const audioSatisfyResult = await this.audioBufferIterator?.tryToSatisfySeek(nextTime);
+            if (this.currentSeekNonce !== nonce) {
+                return;
+            }
+            if (!audioSatisfyResult) {
+                return;
+            }
+            if (audioSatisfyResult.type === 'not-satisfied') {
+                await this.startAudioIterator(nextTime, nonce);
+                return;
+            }
+            toBeScheduled.push(...audioSatisfyResult.buffers);
+        }
+        for (const buffer of toBeScheduled) {
+            if (this.playing) {
+                this.scheduleAudioChunk(buffer.buffer, buffer.timestamp);
+            }
+            else {
+                this.audioChunksForAfterResuming.push({
+                    buffer: buffer.buffer,
+                    timestamp: buffer.timestamp,
+                });
+            }
+        }
+    }
+    async play(time) {
+        if (!this.isReady())
+            return;
+        this.setPlaybackTime(time);
+        this.playing = true;
+        for (const chunk of this.audioChunksForAfterResuming) {
+            this.scheduleAudioChunk(chunk.buffer, chunk.timestamp);
+        }
+        if (this.sharedAudioContext.state === 'suspended') {
+            await this.sharedAudioContext.resume();
+        }
+        this.audioChunksForAfterResuming.length = 0;
+        this.drawDebugOverlay();
+    }
+    pause() {
+        this.playing = false;
+        const toQueue = this.audioBufferIterator?.removeAndReturnAllQueuedAudioNodes();
+        if (toQueue) {
+            for (const chunk of toQueue) {
+                this.audioChunksForAfterResuming.push({
+                    buffer: chunk.buffer,
+                    timestamp: chunk.timestamp,
+                });
+            }
+        }
+        this.drawDebugOverlay();
+    }
+    setMuted(muted) {
+        this.muted = muted;
+        if (this.gainNode) {
+            this.gainNode.gain.value = muted ? 0 : this.currentVolume;
+        }
+    }
+    setVolume(volume) {
+        if (!this.gainNode) {
+            return;
+        }
+        const appliedVolume = Math.max(0, volume);
+        this.currentVolume = appliedVolume;
+        if (!this.muted) {
+            this.gainNode.gain.value = appliedVolume;
+        }
+    }
+    setDebugOverlay(debugOverlay) {
+        this.debugOverlay = debugOverlay;
+    }
+    setPlaybackRate(rate) {
+        this.playbackRate = rate;
+    }
+    setFps(fps) {
+        this.fps = fps;
+    }
+    setLoop(loop) {
+        this.loop = loop;
+    }
+    async dispose() {
+        this.initialized = false;
+        if (this.initializationPromise) {
+            try {
+                // wait for the init to finished
+                // otherwise we might get errors like:
+                // Error: Response stream reader stopped unexpectedly before all requested data was read. from UrlSource
+                await this.initializationPromise;
+            }
+            catch {
+                // Ignore initialization errors during disposal
+            }
+        }
+        this.input?.dispose();
+        this.videoFrameIterator?.destroy();
+        this.videoFrameIterator = null;
+        this.audioBufferIterator?.destroy();
+        this.audioBufferIterator = null;
+    }
+    getPlaybackTime() {
+        return this.sharedAudioContext.currentTime - this.audioSyncAnchor;
+    }
+    setPlaybackTime(time) {
+        this.audioSyncAnchor = this.sharedAudioContext.currentTime - time;
+    }
+    scheduleAudioChunk(buffer, mediaTimestamp) {
+        // TODO: Might already be scheduled, and then the playback rate changes
+        // TODO: Playbackrate does not yet work
+        const targetTime = (mediaTimestamp - (this.trimBefore ?? 0) / this.fps) / this.playbackRate;
+        const delay = targetTime + this.audioSyncAnchor - this.sharedAudioContext.currentTime;
+        const node = this.sharedAudioContext.createBufferSource();
+        node.buffer = buffer;
+        node.playbackRate.value = this.playbackRate;
+        node.connect(this.gainNode);
+        if (delay >= 0) {
+            node.start(targetTime + this.audioSyncAnchor);
+        }
+        else {
+            node.start(this.sharedAudioContext.currentTime, -delay);
+        }
+        this.audioBufferIterator.addQueuedAudioNode(node, mediaTimestamp, buffer);
+        node.onended = () => {
+            return this.audioBufferIterator.removeQueuedAudioNode(node);
+        };
+    }
+    onVideoFrame(callback) {
+        this.onVideoFrameCallback = callback;
+        if (this.initialized && callback && this.canvas) {
+            callback(this.canvas);
+        }
+        return () => {
+            if (this.onVideoFrameCallback === callback) {
+                this.onVideoFrameCallback = undefined;
+            }
+        };
+    }
+    drawDebugOverlay() {
+        if (!this.debugOverlay)
+            return;
+        if (this.context && this.canvas) {
+            drawPreviewOverlay({
+                context: this.context,
+                stats: this.debugStats,
+                audioTime: this.sharedAudioContext.currentTime,
+                audioContextState: this.sharedAudioContext.state,
+                audioSyncAnchor: this.audioSyncAnchor,
+                audioIterator: this.audioBufferIterator,
+                audioChunksForAfterResuming: this.audioChunksForAfterResuming,
+                playing: this.playing,
+            });
+        }
+    }
+}

package/dist/video/props.d.ts

@@ -37,6 +37,7 @@ type OptionalVideoProps = {
     trimBefore: number | undefined;
     toneFrequency: number;
     showInTimeline: boolean;
+    debugOverlay: boolean;
 };
 export type InnerVideoProps = MandatoryVideoProps & OuterVideoProps & OptionalVideoProps;
 export type VideoProps = MandatoryVideoProps & Partial<OuterVideoProps> & Partial<OptionalVideoProps>;

package/dist/video/video-for-preview.d.ts

@@ -1,7 +1,7 @@
 import React from 'react';
 import type { LogLevel, LoopVolumeCurveBehavior, VolumeProp } from 'remotion';
 import type { FallbackOffthreadVideoProps } from './props';
-type NewVideoForPreviewProps = {
+type VideoForPreviewProps = {
     readonly src: string;
     readonly style: React.CSSProperties | undefined;
     readonly playbackRate: number;
@@ -20,6 +20,7 @@ type NewVideoForPreviewProps = {
     readonly disallowFallbackToOffthreadVideo: boolean;
     readonly fallbackOffthreadVideoProps: FallbackOffthreadVideoProps;
     readonly audioStreamIndex: number;
+    readonly debugOverlay: boolean;
 };
-export declare const VideoForPreview: React.FC<NewVideoForPreviewProps>;
+export declare const VideoForPreview: React.FC<VideoForPreviewProps>;
 export {};