@remotion/media 4.0.355 → 4.0.356

package/dist/extract-frame-and-audio.d.ts

@@ -1,6 +1,6 @@
 import type { LogLevel } from 'remotion';
 import type { PcmS16AudioData } from './convert-audiodata/convert-audiodata';
-export declare const extractFrameAndAudio: ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, loop, }: {
+export declare const extractFrameAndAudio: ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, loop, audioStreamIndex, }: {
     src: string;
     timeInSeconds: number;
     logLevel: LogLevel;
@@ -9,8 +9,9 @@ export declare const extractFrameAndAudio: ({ src, timeInSeconds, logLevel, dura
     includeAudio: boolean;
     includeVideo: boolean;
     loop: boolean;
+    audioStreamIndex: number;
 }) => Promise<{
     frame: VideoFrame | null;
     audio: PcmS16AudioData | null;
     durationInSeconds: number | null;
-}>;
+} | "cannot-decode" | "unknown-container-format" | "network-error">;

package/dist/extract-frame-and-audio.js

@@ -1,29 +1,63 @@
 import { extractAudio } from './audio-extraction/extract-audio';
 import { extractFrame } from './video-extraction/extract-frame';
-export const extractFrameAndAudio = async ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, loop, }) => {
-    const [frame, audio] = await Promise.all([
-        includeVideo
-            ? extractFrame({
-                src,
-                timeInSeconds,
-                logLevel,
-                loop,
-            })
-            : null,
-        includeAudio
-            ? extractAudio({
-                src,
-                timeInSeconds,
-                durationInSeconds,
-                logLevel,
-                loop,
-                playbackRate,
-            })
-            : null,
-    ]);
-    return {
-        frame: frame?.toVideoFrame() ?? null,
-        audio: audio?.data ?? null,
-        durationInSeconds: audio?.durationInSeconds ?? null,
-    };
+export const extractFrameAndAudio = async ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, loop, audioStreamIndex, }) => {
+    try {
+        const [frame, audio] = await Promise.all([
+            includeVideo
+                ? extractFrame({
+                    src,
+                    timeInSeconds,
+                    logLevel,
+                    loop,
+                })
+                : null,
+            includeAudio
+                ? extractAudio({
+                    src,
+                    timeInSeconds,
+                    durationInSeconds,
+                    logLevel,
+                    loop,
+                    playbackRate,
+                    audioStreamIndex,
+                })
+                : null,
+        ]);
+        if (frame === 'cannot-decode') {
+            return 'cannot-decode';
+        }
+        if (frame === 'unknown-container-format') {
+            return 'unknown-container-format';
+        }
+        if (audio === 'unknown-container-format') {
+            if (frame !== null) {
+                frame?.close();
+            }
+            return 'unknown-container-format';
+        }
+        if (audio === 'cannot-decode') {
+            if (frame !== null) {
+                frame?.close();
+            }
+            return 'cannot-decode';
+        }
+        return {
+            frame: frame?.toVideoFrame() ?? null,
+            audio: audio?.data ?? null,
+            durationInSeconds: audio?.durationInSeconds ?? null,
+        };
+    }
+    catch (err) {
+        const error = err;
+        if (
+        // Chrome
+        error.message.includes('Failed to fetch') ||
+            // Safari
+            error.message.includes('Load failed') ||
+            // Firefox
+            error.message.includes('NetworkError when attempting to fetch resource')) {
+            return 'network-error';
+        }
+        throw err;
+    }
 };

package/dist/get-sink-weak.d.ts

@@ -2,13 +2,8 @@ import type { LogLevel } from 'remotion';
 import type { GetSink } from './video-extraction/get-frames-since-keyframe';
 export declare const sinkPromises: Record<string, Promise<GetSink>>;
 export declare const getSinkWeak: (src: string, logLevel: LogLevel) => Promise<{
-    video: {
-        sampleSink: import("mediabunny").VideoSampleSink;
-        packetSink: import("mediabunny").EncodedPacketSink;
-    } | null;
-    audio: {
-        sampleSink: import("mediabunny").AudioSampleSink;
-    } | null;
+    getVideo: () => Promise<import("./video-extraction/get-frames-since-keyframe").VideoSinkResult>;
+    getAudio: (index: number) => Promise<import("./video-extraction/get-frames-since-keyframe").AudioSinkResult>;
     actualMatroskaTimestamps: {
         observeTimestamp: (startTime: number) => void;
         getRealTimestamp: (observedTimestamp: number) => number | null;

package/dist/index.d.ts

@@ -1,4 +1,13 @@
-export { Audio as experimental_Audio } from './audio/audio';
-export { AudioProps } from './audio/props';
+import { Audio } from './audio/audio';
+import { Video } from './video/video';
+/**
+ * @deprecated Now just `Audio`
+ */
+export declare const experimental_Audio: import("react").FC<import(".").AudioProps>;
+/**
+ * @deprecated Now just `Video`
+ */
+export declare const experimental_Video: import("react").FC<import(".").VideoProps>;
+export { AudioProps, FallbackHtml5AudioProps } from './audio/props';
 export { VideoProps } from './video/props';
-export { Video as experimental_Video } from './video/video';
+export { Audio, Video };

package/dist/index.js

@@ -1,2 +1,11 @@
-export { Audio as experimental_Audio } from './audio/audio';
-export { Video as experimental_Video } from './video/video';
+import { Audio } from './audio/audio';
+import { Video } from './video/video';
+/**
+ * @deprecated Now just `Audio`
+ */
+export const experimental_Audio = Audio;
+/**
+ * @deprecated Now just `Video`
+ */
+export const experimental_Video = Video;
+export { Audio, Video };

package/dist/video/media-player.d.ts

@@ -0,0 +1,70 @@
+import type { LogLevel } from 'remotion';
+export declare const SEEK_THRESHOLD = 0.05;
+export declare class MediaPlayer {
+    private canvas;
+    private context;
+    private src;
+    private logLevel;
+    private playbackRate;
+    private canvasSink;
+    private videoFrameIterator;
+    private nextFrame;
+    private audioSink;
+    private audioBufferIterator;
+    private queuedAudioNodes;
+    private gainNode;
+    private sharedAudioContext;
+    private audioSyncAnchor;
+    private playing;
+    private muted;
+    private animationFrameId;
+    private videoAsyncId;
+    private initialized;
+    private totalDuration;
+    private isBuffering;
+    private onBufferingChangeCallback?;
+    private audioBufferHealth;
+    private audioIteratorStarted;
+    private readonly HEALTHY_BUFER_THRESHOLD_SECONDS;
+    private onVideoFrameCallback?;
+    constructor({ canvas, src, logLevel, sharedAudioContext, }: {
+        canvas: HTMLCanvasElement;
+        src: string;
+        logLevel: LogLevel;
+        sharedAudioContext: AudioContext;
+    });
+    private input;
+    private isReady;
+    private hasAudio;
+    private isCurrentlyBuffering;
+    initialize(startTime?: number): Promise<void>;
+    private cleanupAudioQueue;
+    private cleanAudioIteratorAndNodes;
+    seekTo(time: number): Promise<void>;
+    play(): Promise<void>;
+    pause(): void;
+    setMuted(muted: boolean): void;
+    setVolume(volume: number): void;
+    setPlaybackRate(rate: number): Promise<void>;
+    dispose(): void;
+    private getPlaybackTime;
+    private getAdjustedTimestamp;
+    private scheduleAudioChunk;
+    onBufferingChange(callback: (isBuffering: boolean) => void): void;
+    onVideoFrame(callback: (frame: CanvasImageSource) => void): void;
+    private canRenderVideo;
+    private startRenderLoop;
+    private stopRenderLoop;
+    private render;
+    private shouldRenderFrame;
+    private drawCurrentFrame;
+    private startAudioIterator;
+    private startVideoIterator;
+    private updateNextFrame;
+    private bufferingStartedAtMs;
+    private minBufferingTimeoutMs;
+    private setBufferingState;
+    private maybeResumeFromBuffering;
+    private maybeForceResumeFromBuffering;
+    private runAudioIterator;
+}

package/dist/video/media-player.js

@@ -0,0 +1,419 @@
+import { ALL_FORMATS, AudioBufferSink, CanvasSink, Input, UrlSource, } from 'mediabunny';
+import { Internals } from 'remotion';
+import { sleep, withTimeout } from './timeout-utils';
+export const SEEK_THRESHOLD = 0.05;
+const AUDIO_BUFFER_TOLERANCE_THRESHOLD = 0.1;
+export class MediaPlayer {
+    constructor({ canvas, src, logLevel, sharedAudioContext, }) {
+        this.canvasSink = null;
+        this.videoFrameIterator = null;
+        this.nextFrame = null;
+        this.audioSink = null;
+        this.audioBufferIterator = null;
+        this.queuedAudioNodes = new Set();
+        this.gainNode = null;
+        // audioDelay = mediaTimestamp + audioSyncAnchor - sharedAudioContext.currentTime
+        this.audioSyncAnchor = 0;
+        this.playing = false;
+        this.muted = false;
+        this.animationFrameId = null;
+        this.videoAsyncId = 0;
+        this.initialized = false;
+        this.totalDuration = 0;
+        // for remotion buffer state
+        this.isBuffering = false;
+        this.audioBufferHealth = 0;
+        this.audioIteratorStarted = false;
+        this.HEALTHY_BUFER_THRESHOLD_SECONDS = 1;
+        this.input = null;
+        this.render = () => {
+            if (this.isBuffering) {
+                this.maybeForceResumeFromBuffering();
+            }
+            if (this.shouldRenderFrame()) {
+                this.drawCurrentFrame();
+            }
+            if (this.playing) {
+                this.animationFrameId = requestAnimationFrame(this.render);
+            }
+            else {
+                this.animationFrameId = null;
+            }
+        };
+        this.startAudioIterator = async (startFromSecond) => {
+            if (!this.hasAudio())
+                return;
+            // Clean up existing audio iterator
+            await this.audioBufferIterator?.return();
+            this.audioIteratorStarted = false;
+            this.audioBufferHealth = 0;
+            try {
+                this.audioBufferIterator = this.audioSink.buffers(startFromSecond);
+                this.runAudioIterator(startFromSecond);
+            }
+            catch (error) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start audio iterator', error);
+            }
+        };
+        this.startVideoIterator = async (timeToSeek) => {
+            if (!this.canvasSink) {
+                return;
+            }
+            this.videoAsyncId++;
+            const currentAsyncId = this.videoAsyncId;
+            await this.videoFrameIterator?.return();
+            this.videoFrameIterator = this.canvasSink.canvases(timeToSeek);
+            try {
+                const firstFrame = (await this.videoFrameIterator.next()).value ?? null;
+                const secondFrame = (await this.videoFrameIterator.next()).value ?? null;
+                if (currentAsyncId !== this.videoAsyncId) {
+                    return;
+                }
+                if (firstFrame) {
+                    Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Drew initial frame ${firstFrame.timestamp.toFixed(3)}s`);
+                    this.context.drawImage(firstFrame.canvas, 0, 0);
+                    if (this.onVideoFrameCallback) {
+                        this.onVideoFrameCallback(this.canvas);
+                    }
+                }
+                this.nextFrame = secondFrame ?? null;
+                if (secondFrame) {
+                    Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Buffered next frame ${secondFrame.timestamp.toFixed(3)}s`);
+                }
+            }
+            catch (error) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start video iterator', error);
+            }
+        };
+        this.updateNextFrame = async () => {
+            if (!this.videoFrameIterator) {
+                return;
+            }
+            try {
+                while (true) {
+                    const newNextFrame = (await this.videoFrameIterator.next()).value ?? null;
+                    if (!newNextFrame) {
+                        break;
+                    }
+                    if (this.getAdjustedTimestamp(newNextFrame.timestamp) <=
+                        this.getPlaybackTime()) {
+                        continue;
+                    }
+                    else {
+                        this.nextFrame = newNextFrame;
+                        Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Buffered next frame ${newNextFrame.timestamp.toFixed(3)}s`);
+                        break;
+                    }
+                }
+            }
+            catch (error) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to update next frame', error);
+            }
+        };
+        this.bufferingStartedAtMs = null;
+        this.minBufferingTimeoutMs = 500;
+        this.runAudioIterator = async (startFromSecond) => {
+            if (!this.hasAudio() || !this.audioBufferIterator)
+                return;
+            try {
+                let totalBufferDuration = 0;
+                let isFirstBuffer = true;
+                this.audioIteratorStarted = true;
+                while (true) {
+                    const BUFFERING_TIMEOUT_MS = 50;
+                    let result;
+                    try {
+                        result = await withTimeout(this.audioBufferIterator.next(), BUFFERING_TIMEOUT_MS, 'Iterator timeout');
+                    }
+                    catch {
+                        this.setBufferingState(true);
+                        await sleep(10);
+                        continue;
+                    }
+                    if (result.done || !result.value) {
+                        break;
+                    }
+                    const { buffer, timestamp, duration } = result.value;
+                    totalBufferDuration += duration;
+                    this.audioBufferHealth = Math.max(0, totalBufferDuration / this.playbackRate);
+                    this.maybeResumeFromBuffering(totalBufferDuration / this.playbackRate);
+                    if (this.playing && !this.muted) {
+                        if (isFirstBuffer) {
+                            this.audioSyncAnchor =
+                                this.sharedAudioContext.currentTime -
+                                    this.getAdjustedTimestamp(timestamp);
+                            isFirstBuffer = false;
+                        }
+                        // if timestamp is less than timeToSeek, skip
+                        // context: for some reason, mediabunny returns buffer at 9.984s, when requested at 10s
+                        if (timestamp < startFromSecond - AUDIO_BUFFER_TOLERANCE_THRESHOLD) {
+                            continue;
+                        }
+                        this.scheduleAudioChunk(buffer, timestamp);
+                    }
+                    if (this.getAdjustedTimestamp(timestamp) - this.getPlaybackTime() >=
+                        1) {
+                        await new Promise((resolve) => {
+                            const check = () => {
+                                if (this.getAdjustedTimestamp(timestamp) - this.getPlaybackTime() <
+                                    1) {
+                                    resolve();
+                                }
+                                else {
+                                    requestAnimationFrame(check);
+                                }
+                            };
+                            check();
+                        });
+                    }
+                }
+            }
+            catch (error) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to run audio iterator', error);
+            }
+        };
+        this.canvas = canvas;
+        this.src = src;
+        this.logLevel = logLevel ?? 'info';
+        this.sharedAudioContext = sharedAudioContext;
+        this.playbackRate = 1;
+        const context = canvas.getContext('2d', {
+            alpha: false,
+            desynchronized: true,
+        });
+        if (!context) {
+            throw new Error('Could not get 2D context from canvas');
+        }
+        this.context = context;
+    }
+    isReady() {
+        return this.initialized && Boolean(this.sharedAudioContext);
+    }
+    hasAudio() {
+        return Boolean(this.audioSink && this.sharedAudioContext && this.gainNode);
+    }
+    isCurrentlyBuffering() {
+        return this.isBuffering && Boolean(this.bufferingStartedAtMs);
+    }
+    async initialize(startTime = 0) {
+        try {
+            const urlSource = new UrlSource(this.src);
+            const input = new Input({
+                source: urlSource,
+                formats: ALL_FORMATS,
+            });
+            this.input = input;
+            this.totalDuration = await input.computeDuration();
+            const videoTrack = await input.getPrimaryVideoTrack();
+            const audioTrack = await input.getPrimaryAudioTrack();
+            if (!videoTrack && !audioTrack) {
+                throw new Error(`No video or audio track found for ${this.src}`);
+            }
+            if (videoTrack) {
+                this.canvasSink = new CanvasSink(videoTrack, {
+                    poolSize: 2,
+                    fit: 'contain',
+                });
+                this.canvas.width = videoTrack.displayWidth;
+                this.canvas.height = videoTrack.displayHeight;
+            }
+            if (audioTrack && this.sharedAudioContext) {
+                this.audioSink = new AudioBufferSink(audioTrack);
+                this.gainNode = this.sharedAudioContext.createGain();
+                this.gainNode.connect(this.sharedAudioContext.destination);
+            }
+            if (this.sharedAudioContext) {
+                this.audioSyncAnchor = this.sharedAudioContext.currentTime - startTime;
+            }
+            this.initialized = true;
+            const mediaTime = startTime * this.playbackRate;
+            await this.startAudioIterator(mediaTime);
+            await this.startVideoIterator(mediaTime);
+            this.startRenderLoop();
+        }
+        catch (error) {
+            Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to initialize', error);
+            throw error;
+        }
+    }
+    cleanupAudioQueue() {
+        for (const node of this.queuedAudioNodes) {
+            node.stop();
+        }
+        this.queuedAudioNodes.clear();
+    }
+    async cleanAudioIteratorAndNodes() {
+        await this.audioBufferIterator?.return();
+        this.audioBufferIterator = null;
+        this.audioIteratorStarted = false;
+        this.audioBufferHealth = 0;
+        this.cleanupAudioQueue();
+    }
+    async seekTo(time) {
+        if (!this.isReady())
+            return;
+        const newTime = Math.max(0, Math.min(time, this.totalDuration));
+        const currentPlaybackTime = this.getPlaybackTime();
+        const isSignificantSeek = Math.abs(newTime - currentPlaybackTime) > SEEK_THRESHOLD;
+        if (isSignificantSeek) {
+            this.nextFrame = null;
+            this.audioSyncAnchor = this.sharedAudioContext.currentTime - newTime;
+            if (this.audioSink) {
+                await this.cleanAudioIteratorAndNodes();
+            }
+            const mediaTime = newTime * this.playbackRate;
+            await this.startAudioIterator(mediaTime);
+            await this.startVideoIterator(mediaTime);
+        }
+        if (!this.playing) {
+            this.render();
+        }
+    }
+    async play() {
+        if (!this.isReady())
+            return;
+        if (!this.playing) {
+            if (this.sharedAudioContext.state === 'suspended') {
+                await this.sharedAudioContext.resume();
+            }
+            this.playing = true;
+            this.startRenderLoop();
+        }
+    }
+    pause() {
+        this.playing = false;
+        this.cleanupAudioQueue();
+        this.stopRenderLoop();
+    }
+    setMuted(muted) {
+        this.muted = muted;
+        if (muted) {
+            this.cleanupAudioQueue();
+        }
+    }
+    setVolume(volume) {
+        if (!this.gainNode) {
+            return;
+        }
+        const appliedVolume = Math.max(0, volume);
+        this.gainNode.gain.value = appliedVolume;
+    }
+    async setPlaybackRate(rate) {
+        if (this.playbackRate === rate)
+            return;
+        this.playbackRate = rate;
+        if (this.hasAudio() && this.playing) {
+            const currentPlaybackTime = this.getPlaybackTime();
+            const mediaTime = currentPlaybackTime * rate;
+            await this.cleanAudioIteratorAndNodes();
+            await this.startAudioIterator(mediaTime);
+        }
+    }
+    dispose() {
+        this.input?.dispose();
+        this.stopRenderLoop();
+        this.videoFrameIterator?.return();
+        this.cleanAudioIteratorAndNodes();
+        this.videoAsyncId++;
+    }
+    getPlaybackTime() {
+        return this.sharedAudioContext.currentTime - this.audioSyncAnchor;
+    }
+    getAdjustedTimestamp(mediaTimestamp) {
+        return mediaTimestamp / this.playbackRate;
+    }
+    scheduleAudioChunk(buffer, mediaTimestamp) {
+        const adjustedTimestamp = this.getAdjustedTimestamp(mediaTimestamp);
+        const targetTime = adjustedTimestamp + this.audioSyncAnchor;
+        const delay = targetTime - this.sharedAudioContext.currentTime;
+        const node = this.sharedAudioContext.createBufferSource();
+        node.buffer = buffer;
+        node.playbackRate.value = this.playbackRate;
+        node.connect(this.gainNode);
+        if (delay >= 0) {
+            node.start(targetTime);
+        }
+        else {
+            node.start(this.sharedAudioContext.currentTime, -delay);
+        }
+        this.queuedAudioNodes.add(node);
+        node.onended = () => this.queuedAudioNodes.delete(node);
+    }
+    onBufferingChange(callback) {
+        this.onBufferingChangeCallback = callback;
+    }
+    onVideoFrame(callback) {
+        this.onVideoFrameCallback = callback;
+        if (this.initialized && callback) {
+            callback(this.canvas);
+        }
+    }
+    canRenderVideo() {
+        return (!this.hasAudio() ||
+            (this.audioIteratorStarted &&
+                this.audioBufferHealth >= this.HEALTHY_BUFER_THRESHOLD_SECONDS));
+    }
+    startRenderLoop() {
+        if (this.animationFrameId !== null) {
+            return;
+        }
+        this.render();
+    }
+    stopRenderLoop() {
+        if (this.animationFrameId !== null) {
+            cancelAnimationFrame(this.animationFrameId);
+            this.animationFrameId = null;
+        }
+    }
+    shouldRenderFrame() {
+        return (!this.isBuffering &&
+            this.canRenderVideo() &&
+            this.nextFrame !== null &&
+            this.getAdjustedTimestamp(this.nextFrame.timestamp) <=
+                this.getPlaybackTime());
+    }
+    drawCurrentFrame() {
+        this.context.drawImage(this.nextFrame.canvas, 0, 0);
+        if (this.onVideoFrameCallback) {
+            this.onVideoFrameCallback(this.canvas);
+        }
+        this.nextFrame = null;
+        this.updateNextFrame();
+    }
+    setBufferingState(isBuffering) {
+        if (this.isBuffering !== isBuffering) {
+            this.isBuffering = isBuffering;
+            if (isBuffering) {
+                this.bufferingStartedAtMs = performance.now();
+                this.onBufferingChangeCallback?.(true);
+            }
+            else {
+                this.bufferingStartedAtMs = null;
+                this.onBufferingChangeCallback?.(false);
+            }
+        }
+    }
+    maybeResumeFromBuffering(currentBufferDuration) {
+        if (!this.isCurrentlyBuffering())
+            return;
+        const now = performance.now();
+        const bufferingDuration = now - this.bufferingStartedAtMs;
+        const minTimeElapsed = bufferingDuration >= this.minBufferingTimeoutMs;
+        const bufferHealthy = currentBufferDuration >= this.HEALTHY_BUFER_THRESHOLD_SECONDS;
+        if (minTimeElapsed && bufferHealthy) {
+            Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Resuming from buffering after ${bufferingDuration}ms - buffer recovered`);
+            this.setBufferingState(false);
+        }
+    }
+    maybeForceResumeFromBuffering() {
+        if (!this.isCurrentlyBuffering())
+            return;
+        const now = performance.now();
+        const bufferingDuration = now - this.bufferingStartedAtMs;
+        const forceTimeout = bufferingDuration > this.minBufferingTimeoutMs * 10;
+        if (forceTimeout) {
+            Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Force resuming from buffering after ${bufferingDuration}ms`);
+            this.setBufferingState(false);
+        }
+    }
+}