@remotion/media 4.0.354 → 4.0.356

package/dist/extract-frame-and-audio.d.ts

@@ -1,6 +1,6 @@
 import type { LogLevel } from 'remotion';
 import type { PcmS16AudioData } from './convert-audiodata/convert-audiodata';
-export declare const extractFrameAndAudio: ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, loop, }: {
+export declare const extractFrameAndAudio: ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, loop, audioStreamIndex, }: {
     src: string;
     timeInSeconds: number;
     logLevel: LogLevel;
@@ -9,8 +9,9 @@ export declare const extractFrameAndAudio: ({ src, timeInSeconds, logLevel, dura
     includeAudio: boolean;
     includeVideo: boolean;
     loop: boolean;
+    audioStreamIndex: number;
 }) => Promise<{
     frame: VideoFrame | null;
     audio: PcmS16AudioData | null;
     durationInSeconds: number | null;
-}>;
+} | "cannot-decode" | "unknown-container-format" | "network-error">;

package/dist/extract-frame-and-audio.js

@@ -1,29 +1,63 @@
 import { extractAudio } from './audio-extraction/extract-audio';
 import { extractFrame } from './video-extraction/extract-frame';
-export const extractFrameAndAudio = async ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, loop, }) => {
-    const [frame, audio] = await Promise.all([
-        includeVideo
-            ? extractFrame({
-                src,
-                timeInSeconds,
-                logLevel,
-                loop,
-            })
-            : null,
-        includeAudio
-            ? extractAudio({
-                src,
-                timeInSeconds,
-                durationInSeconds,
-                logLevel,
-                loop,
-                playbackRate,
-            })
-            : null,
-    ]);
-    return {
-        frame: frame?.toVideoFrame() ?? null,
-        audio: audio?.data ?? null,
-        durationInSeconds: audio?.durationInSeconds ?? null,
-    };
+export const extractFrameAndAudio = async ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, loop, audioStreamIndex, }) => {
+    try {
+        const [frame, audio] = await Promise.all([
+            includeVideo
+                ? extractFrame({
+                    src,
+                    timeInSeconds,
+                    logLevel,
+                    loop,
+                })
+                : null,
+            includeAudio
+                ? extractAudio({
+                    src,
+                    timeInSeconds,
+                    durationInSeconds,
+                    logLevel,
+                    loop,
+                    playbackRate,
+                    audioStreamIndex,
+                })
+                : null,
+        ]);
+        if (frame === 'cannot-decode') {
+            return 'cannot-decode';
+        }
+        if (frame === 'unknown-container-format') {
+            return 'unknown-container-format';
+        }
+        if (audio === 'unknown-container-format') {
+            if (frame !== null) {
+                frame?.close();
+            }
+            return 'unknown-container-format';
+        }
+        if (audio === 'cannot-decode') {
+            if (frame !== null) {
+                frame?.close();
+            }
+            return 'cannot-decode';
+        }
+        return {
+            frame: frame?.toVideoFrame() ?? null,
+            audio: audio?.data ?? null,
+            durationInSeconds: audio?.durationInSeconds ?? null,
+        };
+    }
+    catch (err) {
+        const error = err;
+        if (
+        // Chrome
+        error.message.includes('Failed to fetch') ||
+            // Safari
+            error.message.includes('Load failed') ||
+            // Firefox
+            error.message.includes('NetworkError when attempting to fetch resource')) {
+            return 'network-error';
+        }
+        throw err;
+    }
 };

package/dist/get-sink-weak.d.ts

@@ -0,0 +1,13 @@
+import type { LogLevel } from 'remotion';
+import type { GetSink } from './video-extraction/get-frames-since-keyframe';
+export declare const sinkPromises: Record<string, Promise<GetSink>>;
+export declare const getSinkWeak: (src: string, logLevel: LogLevel) => Promise<{
+    getVideo: () => Promise<import("./video-extraction/get-frames-since-keyframe").VideoSinkResult>;
+    getAudio: (index: number) => Promise<import("./video-extraction/get-frames-since-keyframe").AudioSinkResult>;
+    actualMatroskaTimestamps: {
+        observeTimestamp: (startTime: number) => void;
+        getRealTimestamp: (observedTimestamp: number) => number | null;
+    };
+    isMatroska: boolean;
+    getDuration: () => Promise<number>;
+}>;

package/dist/get-sink-weak.js

@@ -0,0 +1,23 @@
+import { Internals } from 'remotion';
+import { getSinks } from './video-extraction/get-frames-since-keyframe';
+export const sinkPromises = {};
+export const getSinkWeak = async (src, logLevel) => {
+    let promise = sinkPromises[src];
+    if (!promise) {
+        promise = getSinks(src);
+        sinkPromises[src] = promise;
+    }
+    let awaited = await promise;
+    let deferredValue = awaited.deref();
+    if (!deferredValue) {
+        Internals.Log.verbose({
+            logLevel,
+            tag: '@remotion/media',
+        }, `Sink for ${src} was garbage collected, creating new sink`);
+        promise = getSinks(src);
+        sinkPromises[src] = promise;
+        awaited = await promise;
+        deferredValue = awaited.deref();
+    }
+    return deferredValue;
+};

package/dist/index.d.ts

@@ -1,4 +1,13 @@
-export { Audio as experimental_Audio } from './audio/audio';
-export { AudioProps } from './audio/props';
+import { Audio } from './audio/audio';
+import { Video } from './video/video';
+/**
+ * @deprecated Now just `Audio`
+ */
+export declare const experimental_Audio: import("react").FC<import(".").AudioProps>;
+/**
+ * @deprecated Now just `Video`
+ */
+export declare const experimental_Video: import("react").FC<import(".").VideoProps>;
+export { AudioProps, FallbackHtml5AudioProps } from './audio/props';
 export { VideoProps } from './video/props';
-export { Video as experimental_Video } from './video/video';
+export { Audio, Video };

package/dist/index.js

@@ -1,2 +1,11 @@
-export { Audio as experimental_Audio } from './audio/audio';
-export { Video as experimental_Video } from './video/video';
+import { Audio } from './audio/audio';
+import { Video } from './video/video';
+/**
+ * @deprecated Now just `Audio`
+ */
+export const experimental_Audio = Audio;
+/**
+ * @deprecated Now just `Video`
+ */
+export const experimental_Video = Video;
+export { Audio, Video };

package/dist/video/media-player.d.ts

@@ -5,6 +5,7 @@ export declare class MediaPlayer {
     private context;
     private src;
     private logLevel;
+    private playbackRate;
     private canvasSink;
     private videoFrameIterator;
     private nextFrame;
@@ -15,6 +16,7 @@ export declare class MediaPlayer {
     private sharedAudioContext;
     private audioSyncAnchor;
     private playing;
+    private muted;
     private animationFrameId;
     private videoAsyncId;
     private initialized;
@@ -24,6 +26,7 @@ export declare class MediaPlayer {
     private audioBufferHealth;
     private audioIteratorStarted;
     private readonly HEALTHY_BUFER_THRESHOLD_SECONDS;
+    private onVideoFrameCallback?;
     constructor({ canvas, src, logLevel, sharedAudioContext, }: {
         canvas: HTMLCanvasElement;
         src: string;
@@ -40,10 +43,15 @@ export declare class MediaPlayer {
     seekTo(time: number): Promise<void>;
     play(): Promise<void>;
     pause(): void;
+    setMuted(muted: boolean): void;
+    setVolume(volume: number): void;
+    setPlaybackRate(rate: number): Promise<void>;
     dispose(): void;
     private getPlaybackTime;
+    private getAdjustedTimestamp;
     private scheduleAudioChunk;
     onBufferingChange(callback: (isBuffering: boolean) => void): void;
+    onVideoFrame(callback: (frame: CanvasImageSource) => void): void;
     private canRenderVideo;
     private startRenderLoop;
     private stopRenderLoop;

package/dist/video/media-player.js

@@ -2,6 +2,7 @@ import { ALL_FORMATS, AudioBufferSink, CanvasSink, Input, UrlSource, } from 'med
 import { Internals } from 'remotion';
 import { sleep, withTimeout } from './timeout-utils';
 export const SEEK_THRESHOLD = 0.05;
+const AUDIO_BUFFER_TOLERANCE_THRESHOLD = 0.1;
 export class MediaPlayer {
     constructor({ canvas, src, logLevel, sharedAudioContext, }) {
         this.canvasSink = null;
@@ -14,6 +15,7 @@ export class MediaPlayer {
         // audioDelay = mediaTimestamp + audioSyncAnchor - sharedAudioContext.currentTime
         this.audioSyncAnchor = 0;
         this.playing = false;
+        this.muted = false;
         this.animationFrameId = null;
         this.videoAsyncId = 0;
         this.initialized = false;
@@ -38,7 +40,7 @@ export class MediaPlayer {
                 this.animationFrameId = null;
             }
         };
-        this.startAudioIterator = async (timeToSeek) => {
+        this.startAudioIterator = async (startFromSecond) => {
             if (!this.hasAudio())
                 return;
             // Clean up existing audio iterator
@@ -46,8 +48,8 @@ export class MediaPlayer {
             this.audioIteratorStarted = false;
             this.audioBufferHealth = 0;
             try {
-                this.audioBufferIterator = this.audioSink.buffers(timeToSeek);
-                this.runAudioIterator();
+                this.audioBufferIterator = this.audioSink.buffers(startFromSecond);
+                this.runAudioIterator(startFromSecond);
             }
             catch (error) {
                 Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start audio iterator', error);
@@ -70,6 +72,9 @@ export class MediaPlayer {
                 if (firstFrame) {
                     Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Drew initial frame ${firstFrame.timestamp.toFixed(3)}s`);
                     this.context.drawImage(firstFrame.canvas, 0, 0);
+                    if (this.onVideoFrameCallback) {
+                        this.onVideoFrameCallback(this.canvas);
+                    }
                 }
                 this.nextFrame = secondFrame ?? null;
                 if (secondFrame) {
@@ -90,7 +95,8 @@ export class MediaPlayer {
                     if (!newNextFrame) {
                         break;
                     }
-                    if (newNextFrame.timestamp <= this.getPlaybackTime()) {
+                    if (this.getAdjustedTimestamp(newNextFrame.timestamp) <=
+                        this.getPlaybackTime()) {
                         continue;
                     }
                     else {
@@ -106,7 +112,7 @@ export class MediaPlayer {
         };
         this.bufferingStartedAtMs = null;
         this.minBufferingTimeoutMs = 500;
-        this.runAudioIterator = async () => {
+        this.runAudioIterator = async (startFromSecond) => {
             if (!this.hasAudio() || !this.audioBufferIterator)
                 return;
             try {
@@ -129,20 +135,28 @@ export class MediaPlayer {
                     }
                     const { buffer, timestamp, duration } = result.value;
                     totalBufferDuration += duration;
-                    this.audioBufferHealth = Math.max(0, totalBufferDuration);
-                    this.maybeResumeFromBuffering(totalBufferDuration);
-                    if (this.playing) {
+                    this.audioBufferHealth = Math.max(0, totalBufferDuration / this.playbackRate);
+                    this.maybeResumeFromBuffering(totalBufferDuration / this.playbackRate);
+                    if (this.playing && !this.muted) {
                         if (isFirstBuffer) {
                             this.audioSyncAnchor =
-                                this.sharedAudioContext.currentTime - timestamp;
+                                this.sharedAudioContext.currentTime -
+                                    this.getAdjustedTimestamp(timestamp);
                             isFirstBuffer = false;
                         }
+                        // if timestamp is less than timeToSeek, skip
+                        // context: for some reason, mediabunny returns buffer at 9.984s, when requested at 10s
+                        if (timestamp < startFromSecond - AUDIO_BUFFER_TOLERANCE_THRESHOLD) {
+                            continue;
+                        }
                         this.scheduleAudioChunk(buffer, timestamp);
                     }
-                    if (timestamp - this.getPlaybackTime() >= 1) {
+                    if (this.getAdjustedTimestamp(timestamp) - this.getPlaybackTime() >=
+                        1) {
                         await new Promise((resolve) => {
                             const check = () => {
-                                if (timestamp - this.getPlaybackTime() < 1) {
+                                if (this.getAdjustedTimestamp(timestamp) - this.getPlaybackTime() <
+                                    1) {
                                     resolve();
                                 }
                                 else {
@@ -162,6 +176,7 @@ export class MediaPlayer {
         this.src = src;
         this.logLevel = logLevel ?? 'info';
         this.sharedAudioContext = sharedAudioContext;
+        this.playbackRate = 1;
         const context = canvas.getContext('2d', {
             alpha: false,
             desynchronized: true,
@@ -211,8 +226,9 @@ export class MediaPlayer {
                 this.audioSyncAnchor = this.sharedAudioContext.currentTime - startTime;
             }
             this.initialized = true;
-            await this.startAudioIterator(startTime);
-            await this.startVideoIterator(startTime);
+            const mediaTime = startTime * this.playbackRate;
+            await this.startAudioIterator(mediaTime);
+            await this.startVideoIterator(mediaTime);
             this.startRenderLoop();
         }
         catch (error) {
@@ -240,12 +256,14 @@ export class MediaPlayer {
         const currentPlaybackTime = this.getPlaybackTime();
         const isSignificantSeek = Math.abs(newTime - currentPlaybackTime) > SEEK_THRESHOLD;
         if (isSignificantSeek) {
+            this.nextFrame = null;
             this.audioSyncAnchor = this.sharedAudioContext.currentTime - newTime;
             if (this.audioSink) {
                 await this.cleanAudioIteratorAndNodes();
             }
-            await this.startAudioIterator(newTime);
-            await this.startVideoIterator(newTime);
+            const mediaTime = newTime * this.playbackRate;
+            await this.startAudioIterator(mediaTime);
+            await this.startVideoIterator(mediaTime);
         }
         if (!this.playing) {
             this.render();
@@ -267,6 +285,30 @@ export class MediaPlayer {
         this.cleanupAudioQueue();
         this.stopRenderLoop();
     }
+    setMuted(muted) {
+        this.muted = muted;
+        if (muted) {
+            this.cleanupAudioQueue();
+        }
+    }
+    setVolume(volume) {
+        if (!this.gainNode) {
+            return;
+        }
+        const appliedVolume = Math.max(0, volume);
+        this.gainNode.gain.value = appliedVolume;
+    }
+    async setPlaybackRate(rate) {
+        if (this.playbackRate === rate)
+            return;
+        this.playbackRate = rate;
+        if (this.hasAudio() && this.playing) {
+            const currentPlaybackTime = this.getPlaybackTime();
+            const mediaTime = currentPlaybackTime * rate;
+            await this.cleanAudioIteratorAndNodes();
+            await this.startAudioIterator(mediaTime);
+        }
+    }
     dispose() {
         this.input?.dispose();
         this.stopRenderLoop();
@@ -277,11 +319,16 @@ export class MediaPlayer {
     getPlaybackTime() {
         return this.sharedAudioContext.currentTime - this.audioSyncAnchor;
     }
+    getAdjustedTimestamp(mediaTimestamp) {
+        return mediaTimestamp / this.playbackRate;
+    }
     scheduleAudioChunk(buffer, mediaTimestamp) {
-        const targetTime = mediaTimestamp + this.audioSyncAnchor;
+        const adjustedTimestamp = this.getAdjustedTimestamp(mediaTimestamp);
+        const targetTime = adjustedTimestamp + this.audioSyncAnchor;
         const delay = targetTime - this.sharedAudioContext.currentTime;
         const node = this.sharedAudioContext.createBufferSource();
         node.buffer = buffer;
+        node.playbackRate.value = this.playbackRate;
         node.connect(this.gainNode);
         if (delay >= 0) {
             node.start(targetTime);
@@ -295,9 +342,16 @@ export class MediaPlayer {
     onBufferingChange(callback) {
         this.onBufferingChangeCallback = callback;
     }
+    onVideoFrame(callback) {
+        this.onVideoFrameCallback = callback;
+        if (this.initialized && callback) {
+            callback(this.canvas);
+        }
+    }
     canRenderVideo() {
-        return (this.audioIteratorStarted &&
-            this.audioBufferHealth >= this.HEALTHY_BUFER_THRESHOLD_SECONDS);
+        return (!this.hasAudio() ||
+            (this.audioIteratorStarted &&
+                this.audioBufferHealth >= this.HEALTHY_BUFER_THRESHOLD_SECONDS));
     }
     startRenderLoop() {
         if (this.animationFrameId !== null) {
@@ -315,10 +369,14 @@ export class MediaPlayer {
         return (!this.isBuffering &&
             this.canRenderVideo() &&
             this.nextFrame !== null &&
-            this.nextFrame.timestamp <= this.getPlaybackTime());
+            this.getAdjustedTimestamp(this.nextFrame.timestamp) <=
+                this.getPlaybackTime());
     }
     drawCurrentFrame() {
         this.context.drawImage(this.nextFrame.canvas, 0, 0);
+        if (this.onVideoFrameCallback) {
+            this.onVideoFrameCallback(this.canvas);
+        }
         this.nextFrame = null;
         this.updateNextFrame();
     }

package/dist/video/props.d.ts

@@ -1,24 +1,42 @@
 import type { LogLevel, LoopVolumeCurveBehavior, OnVideoFrame, VolumeProp } from 'remotion';
-export type VideoProps = {
+export type FallbackOffthreadVideoProps = {
+    acceptableTimeShiftInSeconds?: number;
+    toneFrequency?: number;
+    transparent?: boolean;
+    toneMapped?: boolean;
+    onError?: (err: Error) => void;
+};
+type MandatoryVideoProps = {
     src: string;
-    className?: string;
-    trimBefore?: number;
-    trimAfter?: number;
-    volume?: VolumeProp;
-    loopVolumeCurveBehavior?: LoopVolumeCurveBehavior;
-    name?: string;
-    pauseWhenBuffering?: boolean;
-    showInTimeline?: boolean;
-    onVideoFrame?: OnVideoFrame;
-    playbackRate?: number;
-    muted?: boolean;
-    delayRenderRetries?: number;
-    delayRenderTimeoutInMilliseconds?: number;
-    style?: React.CSSProperties;
+};
+type OuterVideoProps = {
+    trimBefore: number | undefined;
+    trimAfter: number | undefined;
+};
+type OptionalVideoProps = {
+    className: string | undefined;
+    volume: VolumeProp;
+    loopVolumeCurveBehavior: LoopVolumeCurveBehavior;
+    name: string | undefined;
+    onVideoFrame: OnVideoFrame | undefined;
+    playbackRate: number;
+    muted: boolean;
+    delayRenderRetries: number | null;
+    delayRenderTimeoutInMilliseconds: number | null;
+    style: React.CSSProperties;
     /**
      * @deprecated For internal use only
      */
-    stack?: string;
-    logLevel?: LogLevel;
-    loop?: boolean;
+    stack: string | undefined;
+    logLevel: LogLevel;
+    loop: boolean;
+    audioStreamIndex: number;
+    disallowFallbackToOffthreadVideo: boolean;
+    fallbackOffthreadVideoProps: FallbackOffthreadVideoProps;
+    trimAfter: number | undefined;
+    trimBefore: number | undefined;
+    showInTimeline: boolean;
 };
+export type InnerVideoProps = MandatoryVideoProps & OuterVideoProps & OptionalVideoProps;
+export type VideoProps = MandatoryVideoProps & Partial<OuterVideoProps> & Partial<OptionalVideoProps>;
+export {};

package/dist/video/video-for-preview.d.ts

@@ -1,11 +1,17 @@
 import React from 'react';
-import type { LogLevel } from 'remotion';
-type NewVideoForPreviewProps = {
+import type { LogLevel, LoopVolumeCurveBehavior, OnVideoFrame, VolumeProp } from 'remotion';
+type InnerVideoProps = {
+    readonly className: string | undefined;
+    readonly loop: boolean;
     readonly src: string;
-    readonly style?: React.CSSProperties;
-    readonly playbackRate?: number;
-    readonly logLevel?: LogLevel;
-    readonly className?: string;
+    readonly logLevel: LogLevel;
+    readonly muted: boolean;
+    readonly name: string | undefined;
+    readonly volume: VolumeProp;
+    readonly loopVolumeCurveBehavior: LoopVolumeCurveBehavior;
+    readonly onVideoFrame: OnVideoFrame | undefined;
+    readonly playbackRate: number;
+    readonly style: React.CSSProperties;
 };
-export declare const NewVideoForPreview: React.FC<NewVideoForPreviewProps>;
+export declare const VideoForPreview: React.FC<InnerVideoProps>;
 export {};