@remotion/media 4.0.356 → 4.0.358

package/dist/video/media-player.js

@@ -1,10 +1,12 @@
 import { ALL_FORMATS, AudioBufferSink, CanvasSink, Input, UrlSource, } from 'mediabunny';
 import { Internals } from 'remotion';
+import { getTimeInSeconds } from '../get-time-in-seconds';
+import { isNetworkError } from '../is-network-error';
 import { sleep, withTimeout } from './timeout-utils';
 export const SEEK_THRESHOLD = 0.05;
 const AUDIO_BUFFER_TOLERANCE_THRESHOLD = 0.1;
 export class MediaPlayer {
-    constructor({ canvas, src, logLevel, sharedAudioContext, }) {
+    constructor({ canvas, src, logLevel, sharedAudioContext, loop, trimBefore, trimAfter, playbackRate, audioStreamIndex, fps, }) {
         this.canvasSink = null;
         this.videoFrameIterator = null;
         this.nextFrame = null;
@@ -12,14 +14,17 @@ export class MediaPlayer {
         this.audioBufferIterator = null;
         this.queuedAudioNodes = new Set();
         this.gainNode = null;
-        // audioDelay = mediaTimestamp + audioSyncAnchor - sharedAudioContext.currentTime
+        this.currentVolume = 1;
+        // this is the time difference between Web Audio timeline
+        // and media file timeline
         this.audioSyncAnchor = 0;
         this.playing = false;
         this.muted = false;
+        this.loop = false;
         this.animationFrameId = null;
         this.videoAsyncId = 0;
+        this.audioAsyncId = 0;
         this.initialized = false;
-        this.totalDuration = 0;
         // for remotion buffer state
         this.isBuffering = false;
         this.audioBufferHealth = 0;
@@ -43,13 +48,15 @@ export class MediaPlayer {
         this.startAudioIterator = async (startFromSecond) => {
             if (!this.hasAudio())
                 return;
+            this.audioAsyncId++;
+            const currentAsyncId = this.audioAsyncId;
             // Clean up existing audio iterator
             await this.audioBufferIterator?.return();
             this.audioIteratorStarted = false;
             this.audioBufferHealth = 0;
             try {
                 this.audioBufferIterator = this.audioSink.buffers(startFromSecond);
-                this.runAudioIterator(startFromSecond);
+                this.runAudioIterator(startFromSecond, currentAsyncId);
             }
             catch (error) {
                 Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start audio iterator', error);
@@ -61,7 +68,7 @@ export class MediaPlayer {
             }
             this.videoAsyncId++;
             const currentAsyncId = this.videoAsyncId;
-            await this.videoFrameIterator?.return();
+            this.videoFrameIterator?.return().catch(() => undefined);
             this.videoFrameIterator = this.canvasSink.canvases(timeToSeek);
             try {
                 const firstFrame = (await this.videoFrameIterator.next()).value ?? null;
@@ -69,10 +76,10 @@ export class MediaPlayer {
                 if (currentAsyncId !== this.videoAsyncId) {
                     return;
                 }
-                if (firstFrame) {
+                if (firstFrame && this.context) {
                     Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Drew initial frame ${firstFrame.timestamp.toFixed(3)}s`);
                     this.context.drawImage(firstFrame.canvas, 0, 0);
-                    if (this.onVideoFrameCallback) {
+                    if (this.onVideoFrameCallback && this.canvas) {
                         this.onVideoFrameCallback(this.canvas);
                     }
                 }
@@ -95,8 +102,11 @@ export class MediaPlayer {
                     if (!newNextFrame) {
                         break;
                     }
-                    if (this.getAdjustedTimestamp(newNextFrame.timestamp) <=
-                        this.getPlaybackTime()) {
+                    const playbackTime = this.getPlaybackTime();
+                    if (playbackTime === null) {
+                        continue;
+                    }
+                    if (newNextFrame.timestamp <= playbackTime) {
                         continue;
                     }
                     else {
@@ -112,7 +122,7 @@ export class MediaPlayer {
         };
         this.bufferingStartedAtMs = null;
         this.minBufferingTimeoutMs = 500;
-        this.runAudioIterator = async (startFromSecond) => {
+        this.runAudioIterator = async (startFromSecond, audioAsyncId) => {
             if (!this.hasAudio() || !this.audioBufferIterator)
                 return;
             try {
@@ -120,6 +130,9 @@ export class MediaPlayer {
                 let isFirstBuffer = true;
                 this.audioIteratorStarted = true;
                 while (true) {
+                    if (audioAsyncId !== this.audioAsyncId) {
+                        return;
+                    }
                     const BUFFERING_TIMEOUT_MS = 50;
                     let result;
                     try {
@@ -137,11 +150,10 @@ export class MediaPlayer {
                     totalBufferDuration += duration;
                     this.audioBufferHealth = Math.max(0, totalBufferDuration / this.playbackRate);
                     this.maybeResumeFromBuffering(totalBufferDuration / this.playbackRate);
-                    if (this.playing && !this.muted) {
+                    if (this.playing) {
                         if (isFirstBuffer) {
                             this.audioSyncAnchor =
-                                this.sharedAudioContext.currentTime -
-                                    this.getAdjustedTimestamp(timestamp);
+                                this.sharedAudioContext.currentTime - timestamp;
                             isFirstBuffer = false;
                         }
                         // if timestamp is less than timeToSeek, skip
@@ -151,12 +163,16 @@ export class MediaPlayer {
                         }
                         this.scheduleAudioChunk(buffer, timestamp);
                     }
-                    if (this.getAdjustedTimestamp(timestamp) - this.getPlaybackTime() >=
-                        1) {
+                    const playbackTime = this.getPlaybackTime();
+                    if (playbackTime === null) {
+                        continue;
+                    }
+                    if (timestamp - playbackTime >= 1) {
                         await new Promise((resolve) => {
                             const check = () => {
-                                if (this.getAdjustedTimestamp(timestamp) - this.getPlaybackTime() <
-                                    1) {
+                                const currentPlaybackTime = this.getPlaybackTime();
+                                if (currentPlaybackTime !== null &&
+                                    timestamp - currentPlaybackTime < 1) {
                                     resolve();
                                 }
                                 else {
@@ -172,19 +188,29 @@ export class MediaPlayer {
                 Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to run audio iterator', error);
             }
         };
-        this.canvas = canvas;
+        this.canvas = canvas ?? null;
         this.src = src;
-        this.logLevel = logLevel ?? 'info';
+        this.logLevel = logLevel ?? window.remotion_logLevel;
         this.sharedAudioContext = sharedAudioContext;
-        this.playbackRate = 1;
-        const context = canvas.getContext('2d', {
-            alpha: false,
-            desynchronized: true,
-        });
-        if (!context) {
-            throw new Error('Could not get 2D context from canvas');
+        this.playbackRate = playbackRate;
+        this.loop = loop;
+        this.trimBefore = trimBefore;
+        this.trimAfter = trimAfter;
+        this.audioStreamIndex = audioStreamIndex ?? 0;
+        this.fps = fps;
+        if (canvas) {
+            const context = canvas.getContext('2d', {
+                alpha: true,
+                desynchronized: true,
+            });
+            if (!context) {
+                throw new Error('Could not get 2D context from canvas');
+            }
+            this.context = context;
+        }
+        else {
+            this.context = null;
         }
-        this.context = context;
     }
     isReady() {
         return this.initialized && Boolean(this.sharedAudioContext);
@@ -195,7 +221,7 @@ export class MediaPlayer {
     isCurrentlyBuffering() {
         return this.isBuffering && Boolean(this.bufferingStartedAtMs);
     }
-    async initialize(startTime = 0) {
+    async initialize(startTimeUnresolved) {
         try {
             const urlSource = new UrlSource(this.src);
             const input = new Input({
@@ -203,13 +229,32 @@ export class MediaPlayer {
                 formats: ALL_FORMATS,
             });
             this.input = input;
-            this.totalDuration = await input.computeDuration();
-            const videoTrack = await input.getPrimaryVideoTrack();
-            const audioTrack = await input.getPrimaryAudioTrack();
+            try {
+                await this.input.getFormat();
+            }
+            catch (error) {
+                const err = error;
+                if (isNetworkError(err)) {
+                    throw error;
+                }
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Failed to recognize format for ${this.src}`, error);
+                return { type: 'unknown-container-format' };
+            }
+            const [durationInSeconds, videoTrack, audioTracks] = await Promise.all([
+                input.computeDuration(),
+                input.getPrimaryVideoTrack(),
+                input.getAudioTracks(),
+            ]);
+            this.totalDuration = durationInSeconds;
+            const audioTrack = audioTracks[this.audioStreamIndex] ?? null;
             if (!videoTrack && !audioTrack) {
-                throw new Error(`No video or audio track found for ${this.src}`);
+                return { type: 'no-tracks' };
             }
-            if (videoTrack) {
+            if (videoTrack && this.canvas && this.context) {
+                const canDecode = await videoTrack.canDecode();
+                if (!canDecode) {
+                    return { type: 'cannot-decode' };
+                }
                 this.canvasSink = new CanvasSink(videoTrack, {
                     poolSize: 2,
                     fit: 'contain',
@@ -222,20 +267,47 @@ export class MediaPlayer {
                 this.gainNode = this.sharedAudioContext.createGain();
                 this.gainNode.connect(this.sharedAudioContext.destination);
             }
+            const startTime = getTimeInSeconds({
+                unloopedTimeInSeconds: startTimeUnresolved,
+                playbackRate: this.playbackRate,
+                loop: this.loop,
+                trimBefore: this.trimBefore,
+                trimAfter: this.trimAfter,
+                mediaDurationInSeconds: this.totalDuration,
+                fps: this.fps,
+                ifNoMediaDuration: 'infinity',
+                src: this.src,
+            });
+            if (startTime === null) {
+                this.clearCanvas();
+                return { type: 'success', durationInSeconds: this.totalDuration };
+            }
             if (this.sharedAudioContext) {
                 this.audioSyncAnchor = this.sharedAudioContext.currentTime - startTime;
             }
             this.initialized = true;
-            const mediaTime = startTime * this.playbackRate;
-            await this.startAudioIterator(mediaTime);
-            await this.startVideoIterator(mediaTime);
+            await Promise.all([
+                this.startAudioIterator(startTime),
+                this.startVideoIterator(startTime),
+            ]);
             this.startRenderLoop();
+            return { type: 'success', durationInSeconds };
         }
         catch (error) {
+            const err = error;
+            if (isNetworkError(err)) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Network/CORS error for ${this.src}`, err);
+                return { type: 'network-error' };
+            }
             Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to initialize', error);
             throw error;
         }
     }
+    clearCanvas() {
+        if (this.context && this.canvas) {
+            this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
+        }
+    }
     cleanupAudioQueue() {
         for (const node of this.queuedAudioNodes) {
             node.stop();
@@ -252,18 +324,35 @@ export class MediaPlayer {
     async seekTo(time) {
         if (!this.isReady())
             return;
-        const newTime = Math.max(0, Math.min(time, this.totalDuration));
+        const newTime = getTimeInSeconds({
+            unloopedTimeInSeconds: time,
+            playbackRate: this.playbackRate,
+            loop: this.loop,
+            trimBefore: this.trimBefore,
+            trimAfter: this.trimAfter,
+            mediaDurationInSeconds: this.totalDuration ?? null,
+            fps: this.fps,
+            ifNoMediaDuration: 'infinity',
+            src: this.src,
+        });
+        if (newTime === null) {
+            this.clearCanvas();
+            await this.cleanAudioIteratorAndNodes();
+            return;
+        }
         const currentPlaybackTime = this.getPlaybackTime();
-        const isSignificantSeek = Math.abs(newTime - currentPlaybackTime) > SEEK_THRESHOLD;
+        const isSignificantSeek = currentPlaybackTime === null ||
+            Math.abs(newTime - currentPlaybackTime) > SEEK_THRESHOLD;
         if (isSignificantSeek) {
             this.nextFrame = null;
             this.audioSyncAnchor = this.sharedAudioContext.currentTime - newTime;
             if (this.audioSink) {
                 await this.cleanAudioIteratorAndNodes();
             }
-            const mediaTime = newTime * this.playbackRate;
-            await this.startAudioIterator(mediaTime);
-            await this.startVideoIterator(mediaTime);
+            await Promise.all([
+                this.startAudioIterator(newTime),
+                this.startVideoIterator(newTime),
+            ]);
         }
         if (!this.playing) {
             this.render();
@@ -287,8 +376,8 @@ export class MediaPlayer {
     }
     setMuted(muted) {
         this.muted = muted;
-        if (muted) {
-            this.cleanupAudioQueue();
+        if (this.gainNode) {
+            this.gainNode.gain.value = muted ? 0 : this.currentVolume;
         }
     }
     setVolume(volume) {
@@ -296,18 +385,19 @@ export class MediaPlayer {
             return;
         }
         const appliedVolume = Math.max(0, volume);
-        this.gainNode.gain.value = appliedVolume;
+        this.currentVolume = appliedVolume;
+        if (!this.muted) {
+            this.gainNode.gain.value = appliedVolume;
+        }
     }
-    async setPlaybackRate(rate) {
-        if (this.playbackRate === rate)
-            return;
+    setPlaybackRate(rate) {
         this.playbackRate = rate;
-        if (this.hasAudio() && this.playing) {
-            const currentPlaybackTime = this.getPlaybackTime();
-            const mediaTime = currentPlaybackTime * rate;
-            await this.cleanAudioIteratorAndNodes();
-            await this.startAudioIterator(mediaTime);
-        }
+    }
+    setFps(fps) {
+        this.fps = fps;
+    }
+    setLoop(loop) {
+        this.loop = loop;
     }
     dispose() {
         this.input?.dispose();
@@ -319,12 +409,8 @@ export class MediaPlayer {
     getPlaybackTime() {
         return this.sharedAudioContext.currentTime - this.audioSyncAnchor;
     }
-    getAdjustedTimestamp(mediaTimestamp) {
-        return mediaTimestamp / this.playbackRate;
-    }
     scheduleAudioChunk(buffer, mediaTimestamp) {
-        const adjustedTimestamp = this.getAdjustedTimestamp(mediaTimestamp);
-        const targetTime = adjustedTimestamp + this.audioSyncAnchor;
+        const targetTime = mediaTimestamp + this.audioSyncAnchor;
         const delay = targetTime - this.sharedAudioContext.currentTime;
         const node = this.sharedAudioContext.createBufferSource();
         node.buffer = buffer;
@@ -341,12 +427,22 @@ export class MediaPlayer {
     }
     onBufferingChange(callback) {
         this.onBufferingChangeCallback = callback;
+        return () => {
+            if (this.onBufferingChangeCallback === callback) {
+                this.onBufferingChangeCallback = undefined;
+            }
+        };
     }
     onVideoFrame(callback) {
         this.onVideoFrameCallback = callback;
-        if (this.initialized && callback) {
+        if (this.initialized && callback && this.canvas) {
             callback(this.canvas);
         }
+        return () => {
+            if (this.onVideoFrameCallback === callback) {
+                this.onVideoFrameCallback = undefined;
+            }
+        };
     }
     canRenderVideo() {
         return (!this.hasAudio() ||
@@ -366,15 +462,20 @@ export class MediaPlayer {
         }
     }
     shouldRenderFrame() {
+        const playbackTime = this.getPlaybackTime();
+        if (playbackTime === null) {
+            return false;
+        }
         return (!this.isBuffering &&
             this.canRenderVideo() &&
             this.nextFrame !== null &&
-            this.getAdjustedTimestamp(this.nextFrame.timestamp) <=
-                this.getPlaybackTime());
+            this.nextFrame.timestamp <= playbackTime);
     }
     drawCurrentFrame() {
-        this.context.drawImage(this.nextFrame.canvas, 0, 0);
-        if (this.onVideoFrameCallback) {
+        if (this.context && this.nextFrame) {
+            this.context.drawImage(this.nextFrame.canvas, 0, 0);
+        }
+        if (this.onVideoFrameCallback && this.canvas) {
             this.onVideoFrameCallback(this.canvas);
         }
         this.nextFrame = null;

package/dist/video/props.d.ts

@@ -35,6 +35,7 @@ type OptionalVideoProps = {
     fallbackOffthreadVideoProps: FallbackOffthreadVideoProps;
     trimAfter: number | undefined;
     trimBefore: number | undefined;
+    toneFrequency: number;
     showInTimeline: boolean;
 };
 export type InnerVideoProps = MandatoryVideoProps & OuterVideoProps & OptionalVideoProps;

package/dist/video/video-for-preview.d.ts

@@ -1,17 +1,25 @@
 import React from 'react';
-import type { LogLevel, LoopVolumeCurveBehavior, OnVideoFrame, VolumeProp } from 'remotion';
-type InnerVideoProps = {
-    readonly className: string | undefined;
-    readonly loop: boolean;
+import type { LogLevel, LoopVolumeCurveBehavior, VolumeProp } from 'remotion';
+import type { FallbackOffthreadVideoProps } from './props';
+type NewVideoForPreviewProps = {
     readonly src: string;
+    readonly style: React.CSSProperties | undefined;
+    readonly playbackRate: number;
     readonly logLevel: LogLevel;
+    readonly className: string | undefined;
     readonly muted: boolean;
-    readonly name: string | undefined;
     readonly volume: VolumeProp;
     readonly loopVolumeCurveBehavior: LoopVolumeCurveBehavior;
-    readonly onVideoFrame: OnVideoFrame | undefined;
-    readonly playbackRate: number;
-    readonly style: React.CSSProperties;
+    readonly onVideoFrame: undefined | ((frame: CanvasImageSource) => void);
+    readonly showInTimeline: boolean;
+    readonly loop: boolean;
+    readonly name: string | undefined;
+    readonly trimAfter: number | undefined;
+    readonly trimBefore: number | undefined;
+    readonly stack: string | null;
+    readonly disallowFallbackToOffthreadVideo: boolean;
+    readonly fallbackOffthreadVideoProps: FallbackOffthreadVideoProps;
+    readonly audioStreamIndex: number;
 };
-export declare const VideoForPreview: React.FC<InnerVideoProps>;
+export declare const VideoForPreview: React.FC<NewVideoForPreviewProps>;
 export {};