@remotion/media 4.0.356 → 4.0.357

package/dist/video/media-player.js

@@ -1,10 +1,12 @@
 import { ALL_FORMATS, AudioBufferSink, CanvasSink, Input, UrlSource, } from 'mediabunny';
 import { Internals } from 'remotion';
+import { isNetworkError } from '../is-network-error';
+import { resolvePlaybackTime } from './resolve-playback-time';
 import { sleep, withTimeout } from './timeout-utils';
 export const SEEK_THRESHOLD = 0.05;
 const AUDIO_BUFFER_TOLERANCE_THRESHOLD = 0.1;
 export class MediaPlayer {
-    constructor({ canvas, src, logLevel, sharedAudioContext, }) {
+    constructor({ canvas, src, logLevel, sharedAudioContext, loop, trimBeforeSeconds, trimAfterSeconds, playbackRate, audioStreamIndex, }) {
         this.canvasSink = null;
         this.videoFrameIterator = null;
         this.nextFrame = null;
@@ -16,10 +18,11 @@ export class MediaPlayer {
         this.audioSyncAnchor = 0;
         this.playing = false;
         this.muted = false;
+        this.loop = false;
         this.animationFrameId = null;
         this.videoAsyncId = 0;
+        this.audioAsyncId = 0;
         this.initialized = false;
-        this.totalDuration = 0;
         // for remotion buffer state
         this.isBuffering = false;
         this.audioBufferHealth = 0;
@@ -43,13 +46,15 @@ export class MediaPlayer {
         this.startAudioIterator = async (startFromSecond) => {
             if (!this.hasAudio())
                 return;
+            this.audioAsyncId++;
+            const currentAsyncId = this.audioAsyncId;
             // Clean up existing audio iterator
             await this.audioBufferIterator?.return();
             this.audioIteratorStarted = false;
             this.audioBufferHealth = 0;
             try {
                 this.audioBufferIterator = this.audioSink.buffers(startFromSecond);
-                this.runAudioIterator(startFromSecond);
+                this.runAudioIterator(startFromSecond, currentAsyncId);
             }
             catch (error) {
                 Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start audio iterator', error);
@@ -61,7 +66,7 @@ export class MediaPlayer {
             }
             this.videoAsyncId++;
             const currentAsyncId = this.videoAsyncId;
-            await this.videoFrameIterator?.return();
+            this.videoFrameIterator?.return().catch(() => undefined);
             this.videoFrameIterator = this.canvasSink.canvases(timeToSeek);
             try {
                 const firstFrame = (await this.videoFrameIterator.next()).value ?? null;
@@ -69,10 +74,10 @@ export class MediaPlayer {
                 if (currentAsyncId !== this.videoAsyncId) {
                     return;
                 }
-                if (firstFrame) {
+                if (firstFrame && this.context) {
                     Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Drew initial frame ${firstFrame.timestamp.toFixed(3)}s`);
                     this.context.drawImage(firstFrame.canvas, 0, 0);
-                    if (this.onVideoFrameCallback) {
+                    if (this.onVideoFrameCallback && this.canvas) {
                         this.onVideoFrameCallback(this.canvas);
                     }
                 }
@@ -95,8 +100,7 @@ export class MediaPlayer {
                     if (!newNextFrame) {
                         break;
                     }
-                    if (this.getAdjustedTimestamp(newNextFrame.timestamp) <=
-                        this.getPlaybackTime()) {
+                    if (newNextFrame.timestamp <= this.getPlaybackTime()) {
                         continue;
                     }
                     else {
@@ -112,7 +116,7 @@ export class MediaPlayer {
         };
         this.bufferingStartedAtMs = null;
         this.minBufferingTimeoutMs = 500;
-        this.runAudioIterator = async (startFromSecond) => {
+        this.runAudioIterator = async (startFromSecond, audioAsyncId) => {
             if (!this.hasAudio() || !this.audioBufferIterator)
                 return;
             try {
@@ -120,6 +124,9 @@ export class MediaPlayer {
                 let isFirstBuffer = true;
                 this.audioIteratorStarted = true;
                 while (true) {
+                    if (audioAsyncId !== this.audioAsyncId) {
+                        return;
+                    }
                     const BUFFERING_TIMEOUT_MS = 50;
                     let result;
                     try {
@@ -140,8 +147,7 @@ export class MediaPlayer {
                     if (this.playing && !this.muted) {
                         if (isFirstBuffer) {
                             this.audioSyncAnchor =
-                                this.sharedAudioContext.currentTime -
-                                    this.getAdjustedTimestamp(timestamp);
+                                this.sharedAudioContext.currentTime - timestamp;
                             isFirstBuffer = false;
                         }
                         // if timestamp is less than timeToSeek, skip
@@ -151,12 +157,10 @@ export class MediaPlayer {
                         }
                         this.scheduleAudioChunk(buffer, timestamp);
                     }
-                    if (this.getAdjustedTimestamp(timestamp) - this.getPlaybackTime() >=
-                        1) {
+                    if (timestamp - this.getPlaybackTime() >= 1) {
                         await new Promise((resolve) => {
                             const check = () => {
-                                if (this.getAdjustedTimestamp(timestamp) - this.getPlaybackTime() <
-                                    1) {
+                                if (timestamp - this.getPlaybackTime() < 1) {
                                     resolve();
                                 }
                                 else {
@@ -172,19 +176,28 @@ export class MediaPlayer {
                 Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to run audio iterator', error);
             }
         };
-        this.canvas = canvas;
+        this.canvas = canvas ?? null;
         this.src = src;
-        this.logLevel = logLevel ?? 'info';
+        this.logLevel = logLevel ?? window.remotion_logLevel;
         this.sharedAudioContext = sharedAudioContext;
-        this.playbackRate = 1;
-        const context = canvas.getContext('2d', {
-            alpha: false,
-            desynchronized: true,
-        });
-        if (!context) {
-            throw new Error('Could not get 2D context from canvas');
+        this.playbackRate = playbackRate;
+        this.loop = loop;
+        this.trimBeforeSeconds = trimBeforeSeconds;
+        this.trimAfterSeconds = trimAfterSeconds;
+        this.audioStreamIndex = audioStreamIndex ?? 0;
+        if (canvas) {
+            const context = canvas.getContext('2d', {
+                alpha: false,
+                desynchronized: true,
+            });
+            if (!context) {
+                throw new Error('Could not get 2D context from canvas');
+            }
+            this.context = context;
+        }
+        else {
+            this.context = null;
         }
-        this.context = context;
     }
     isReady() {
         return this.initialized && Boolean(this.sharedAudioContext);
@@ -195,7 +208,7 @@ export class MediaPlayer {
     isCurrentlyBuffering() {
         return this.isBuffering && Boolean(this.bufferingStartedAtMs);
     }
-    async initialize(startTime = 0) {
+    async initialize(startTimeUnresolved) {
         try {
             const urlSource = new UrlSource(this.src);
             const input = new Input({
@@ -203,13 +216,32 @@ export class MediaPlayer {
                 formats: ALL_FORMATS,
             });
             this.input = input;
-            this.totalDuration = await input.computeDuration();
-            const videoTrack = await input.getPrimaryVideoTrack();
-            const audioTrack = await input.getPrimaryAudioTrack();
+            try {
+                await this.input.getFormat();
+            }
+            catch (error) {
+                const err = error;
+                if (isNetworkError(err)) {
+                    throw error;
+                }
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Failed to recognize format for ${this.src}`, error);
+                return { type: 'unknown-container-format' };
+            }
+            const [duration, videoTrack, audioTracks] = await Promise.all([
+                input.computeDuration(),
+                input.getPrimaryVideoTrack(),
+                input.getAudioTracks(),
+            ]);
+            this.totalDuration = duration;
+            const audioTrack = audioTracks[this.audioStreamIndex] ?? null;
             if (!videoTrack && !audioTrack) {
-                throw new Error(`No video or audio track found for ${this.src}`);
+                return { type: 'no-tracks' };
             }
-            if (videoTrack) {
+            if (videoTrack && this.canvas && this.context) {
+                const canDecode = await videoTrack.canDecode();
+                if (!canDecode) {
+                    return { type: 'cannot-decode' };
+                }
                 this.canvasSink = new CanvasSink(videoTrack, {
                     poolSize: 2,
                     fit: 'contain',
@@ -222,16 +254,31 @@ export class MediaPlayer {
                 this.gainNode = this.sharedAudioContext.createGain();
                 this.gainNode.connect(this.sharedAudioContext.destination);
             }
+            const startTime = resolvePlaybackTime({
+                absolutePlaybackTimeInSeconds: startTimeUnresolved,
+                playbackRate: this.playbackRate,
+                loop: this.loop,
+                trimBeforeInSeconds: this.trimBeforeSeconds,
+                trimAfterInSeconds: this.trimAfterSeconds,
+                mediaDurationInSeconds: this.totalDuration,
+            });
             if (this.sharedAudioContext) {
                 this.audioSyncAnchor = this.sharedAudioContext.currentTime - startTime;
             }
             this.initialized = true;
-            const mediaTime = startTime * this.playbackRate;
-            await this.startAudioIterator(mediaTime);
-            await this.startVideoIterator(mediaTime);
+            await Promise.all([
+                this.startAudioIterator(startTime),
+                this.startVideoIterator(startTime),
+            ]);
             this.startRenderLoop();
+            return { type: 'success' };
         }
         catch (error) {
+            const err = error;
+            if (isNetworkError(err)) {
+                Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Network/CORS error for ${this.src}`, err);
+                return { type: 'network-error' };
+            }
             Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to initialize', error);
             throw error;
         }
@@ -252,7 +299,14 @@ export class MediaPlayer {
     async seekTo(time) {
         if (!this.isReady())
             return;
-        const newTime = Math.max(0, Math.min(time, this.totalDuration));
+        const newTime = resolvePlaybackTime({
+            absolutePlaybackTimeInSeconds: time,
+            playbackRate: this.playbackRate,
+            loop: this.loop,
+            trimBeforeInSeconds: this.trimBeforeSeconds,
+            trimAfterInSeconds: this.trimAfterSeconds,
+            mediaDurationInSeconds: this.totalDuration,
+        });
         const currentPlaybackTime = this.getPlaybackTime();
         const isSignificantSeek = Math.abs(newTime - currentPlaybackTime) > SEEK_THRESHOLD;
         if (isSignificantSeek) {
@@ -261,9 +315,10 @@ export class MediaPlayer {
             if (this.audioSink) {
                 await this.cleanAudioIteratorAndNodes();
             }
-            const mediaTime = newTime * this.playbackRate;
-            await this.startAudioIterator(mediaTime);
-            await this.startVideoIterator(mediaTime);
+            await Promise.all([
+                this.startAudioIterator(newTime),
+                this.startVideoIterator(newTime),
+            ]);
         }
         if (!this.playing) {
             this.render();
@@ -298,16 +353,11 @@ export class MediaPlayer {
         const appliedVolume = Math.max(0, volume);
         this.gainNode.gain.value = appliedVolume;
     }
-    async setPlaybackRate(rate) {
-        if (this.playbackRate === rate)
-            return;
+    setPlaybackRate(rate) {
         this.playbackRate = rate;
-        if (this.hasAudio() && this.playing) {
-            const currentPlaybackTime = this.getPlaybackTime();
-            const mediaTime = currentPlaybackTime * rate;
-            await this.cleanAudioIteratorAndNodes();
-            await this.startAudioIterator(mediaTime);
-        }
+    }
+    setLoop(loop) {
+        this.loop = loop;
     }
     dispose() {
         this.input?.dispose();
@@ -317,14 +367,18 @@ export class MediaPlayer {
         this.videoAsyncId++;
     }
     getPlaybackTime() {
-        return this.sharedAudioContext.currentTime - this.audioSyncAnchor;
-    }
-    getAdjustedTimestamp(mediaTimestamp) {
-        return mediaTimestamp / this.playbackRate;
+        const absoluteTime = this.sharedAudioContext.currentTime - this.audioSyncAnchor;
+        return resolvePlaybackTime({
+            absolutePlaybackTimeInSeconds: absoluteTime,
+            playbackRate: this.playbackRate,
+            loop: this.loop,
+            trimBeforeInSeconds: this.trimBeforeSeconds,
+            trimAfterInSeconds: this.trimAfterSeconds,
+            mediaDurationInSeconds: this.totalDuration,
+        });
     }
     scheduleAudioChunk(buffer, mediaTimestamp) {
-        const adjustedTimestamp = this.getAdjustedTimestamp(mediaTimestamp);
-        const targetTime = adjustedTimestamp + this.audioSyncAnchor;
+        const targetTime = mediaTimestamp + this.audioSyncAnchor;
         const delay = targetTime - this.sharedAudioContext.currentTime;
         const node = this.sharedAudioContext.createBufferSource();
         node.buffer = buffer;
@@ -341,12 +395,22 @@ export class MediaPlayer {
     }
     onBufferingChange(callback) {
         this.onBufferingChangeCallback = callback;
+        return () => {
+            if (this.onBufferingChangeCallback === callback) {
+                this.onBufferingChangeCallback = undefined;
+            }
+        };
     }
     onVideoFrame(callback) {
         this.onVideoFrameCallback = callback;
-        if (this.initialized && callback) {
+        if (this.initialized && callback && this.canvas) {
             callback(this.canvas);
         }
+        return () => {
+            if (this.onVideoFrameCallback === callback) {
+                this.onVideoFrameCallback = undefined;
+            }
+        };
     }
     canRenderVideo() {
         return (!this.hasAudio() ||
@@ -369,12 +433,13 @@ export class MediaPlayer {
         return (!this.isBuffering &&
             this.canRenderVideo() &&
             this.nextFrame !== null &&
-            this.getAdjustedTimestamp(this.nextFrame.timestamp) <=
-                this.getPlaybackTime());
+            this.nextFrame.timestamp <= this.getPlaybackTime());
     }
     drawCurrentFrame() {
-        this.context.drawImage(this.nextFrame.canvas, 0, 0);
-        if (this.onVideoFrameCallback) {
+        if (this.context && this.nextFrame) {
+            this.context.drawImage(this.nextFrame.canvas, 0, 0);
+        }
+        if (this.onVideoFrameCallback && this.canvas) {
             this.onVideoFrameCallback(this.canvas);
         }
         this.nextFrame = null;

package/dist/video/props.d.ts

@@ -35,6 +35,7 @@ type OptionalVideoProps = {
     fallbackOffthreadVideoProps: FallbackOffthreadVideoProps;
     trimAfter: number | undefined;
     trimBefore: number | undefined;
+    toneFrequency: number;
     showInTimeline: boolean;
 };
 export type InnerVideoProps = MandatoryVideoProps & OuterVideoProps & OptionalVideoProps;

package/dist/video/resolve-playback-time.d.ts

@@ -0,0 +1,8 @@
+export declare const resolvePlaybackTime: ({ absolutePlaybackTimeInSeconds, playbackRate, loop, trimBeforeInSeconds, trimAfterInSeconds, mediaDurationInSeconds, }: {
+    absolutePlaybackTimeInSeconds: number;
+    playbackRate: number;
+    loop: boolean;
+    trimBeforeInSeconds: number | undefined;
+    trimAfterInSeconds: number | undefined;
+    mediaDurationInSeconds: number | undefined;
+}) => number;

package/dist/video/resolve-playback-time.js

@@ -0,0 +1,22 @@
+export const resolvePlaybackTime = ({ absolutePlaybackTimeInSeconds, playbackRate, loop, trimBeforeInSeconds, trimAfterInSeconds, mediaDurationInSeconds, }) => {
+    const loopAfterPreliminary = loop
+        ? Math.min(trimAfterInSeconds ?? Infinity, mediaDurationInSeconds ?? Infinity)
+        : Infinity;
+    const loopAfterConsideringTrimBefore = loopAfterPreliminary - (trimBeforeInSeconds ?? 0);
+    const loopAfterConsideringPlaybackRate = loopAfterConsideringTrimBefore / playbackRate;
+    const timeConsideringLoop = absolutePlaybackTimeInSeconds % loopAfterConsideringPlaybackRate;
+    const time = timeConsideringLoop * playbackRate + (trimBeforeInSeconds ?? 0);
+    if (Number.isNaN(time)) {
+        // eslint-disable-next-line no-console
+        console.log({
+            absolutePlaybackTimeInSeconds,
+            playbackRate,
+            loop,
+            trimBeforeInSeconds,
+            trimAfterInSeconds,
+            mediaDurationInSeconds,
+        });
+        throw new Error('Time is NaN');
+    }
+    return time;
+};

package/dist/video/video-for-preview.d.ts

@@ -1,5 +1,6 @@
 import React from 'react';
 import type { LogLevel, LoopVolumeCurveBehavior, OnVideoFrame, VolumeProp } from 'remotion';
+import type { FallbackOffthreadVideoProps } from './props';
 type InnerVideoProps = {
     readonly className: string | undefined;
     readonly loop: boolean;
@@ -12,6 +13,13 @@ type InnerVideoProps = {
     readonly onVideoFrame: OnVideoFrame | undefined;
     readonly playbackRate: number;
     readonly style: React.CSSProperties;
+    readonly showInTimeline: boolean;
+    readonly trimAfter: number | undefined;
+    readonly trimBefore: number | undefined;
+    readonly stack: string | null;
+    readonly disallowFallbackToOffthreadVideo: boolean;
+    readonly fallbackOffthreadVideoProps: FallbackOffthreadVideoProps;
+    readonly audioStreamIndex: number;
 };
 export declare const VideoForPreview: React.FC<InnerVideoProps>;
 export {};