@remotion/media 4.0.372 → 4.0.374

package/dist/audio/audio-for-preview.js

@@ -13,7 +13,7 @@ const AudioForPreviewAssertedShowing = ({ src, playbackRate, logLevel, muted, vo
     const [mediaPlayerReady, setMediaPlayerReady] = useState(false);
     const [shouldFallbackToNativeAudio, setShouldFallbackToNativeAudio] = useState(false);
     const [playing] = Timeline.usePlayingState();
-    const timelineContext = useContext(Timeline.TimelineContext);
+    const timelineContext = useContext(Internals.TimelineContext);
     const globalPlaybackRate = timelineContext.playbackRate;
     const sharedAudioContext = useContext(SharedAudioContext);
     const buffer = useBufferState();

package/dist/audio-extraction/extract-audio.js

@@ -44,6 +44,7 @@ const extractAudioInternal = async ({ src, timeInSeconds: unloopedTimeInSeconds,
     const durationInSeconds = durationNotYetApplyingPlaybackRate * playbackRate;
     const samples = await sampleIterator.getSamples(timeInSeconds, durationInSeconds);
     audioManager.logOpenFrames();
+    const trimStartToleranceInSeconds = 0.002;
     const audioDataArray = [];
     for (let i = 0; i < samples.length; i++) {
         const sample = samples[i];
@@ -65,7 +66,8 @@ const extractAudioInternal = async ({ src, timeInSeconds: unloopedTimeInSeconds,
         let trimEndInSeconds = 0;
         if (isFirstSample) {
             trimStartInSeconds = timeInSeconds - sample.timestamp;
-            if (trimStartInSeconds < 0 && trimStartInSeconds > -1e-10) {
+            if (trimStartInSeconds < 0 &&
+                trimStartInSeconds > -trimStartToleranceInSeconds) {
                 trimStartInSeconds = 0;
             }
             if (trimStartInSeconds < 0) {

package/dist/esm/index.mjs

@@ -994,11 +994,6 @@ class MediaPlayer {
     if (currentPlaybackTime === newTime) {
       return;
     }
-    const newAudioSyncAnchor = this.sharedAudioContext.currentTime - newTime / (this.playbackRate * this.globalPlaybackRate);
-    const diff = Math.abs(newAudioSyncAnchor - this.audioSyncAnchor);
-    if (diff > 0.04) {
-      this.setPlaybackTime(newTime, this.playbackRate * this.globalPlaybackRate);
-    }
     await this.videoIteratorManager?.seek({
       newTime,
       nonce
@@ -1362,7 +1357,7 @@ var AudioForPreviewAssertedShowing = ({
   const [mediaPlayerReady, setMediaPlayerReady] = useState2(false);
   const [shouldFallbackToNativeAudio, setShouldFallbackToNativeAudio] = useState2(false);
   const [playing] = Timeline.usePlayingState();
-  const timelineContext = useContext2(Timeline.TimelineContext);
+  const timelineContext = useContext2(Internals6.TimelineContext);
   const globalPlaybackRate = timelineContext.playbackRate;
   const sharedAudioContext = useContext2(SharedAudioContext);
   const buffer = useBufferState();
@@ -2270,19 +2265,26 @@ var makeKeyframeBank = ({
   };
   const getFrameFromTimestamp = async (timestampInSeconds) => {
     lastUsed = Date.now();
-    if (timestampInSeconds < startTimestampInSeconds) {
-      return Promise.reject(new Error(`Timestamp is before start timestamp (requested: ${timestampInSeconds}sec, start: ${startTimestampInSeconds})`));
+    const maxClampToleranceInSeconds = 0.1;
+    let adjustedTimestamp = timestampInSeconds;
+    if (roundTo4Digits(timestampInSeconds) < roundTo4Digits(startTimestampInSeconds)) {
+      const differenceInSeconds = startTimestampInSeconds - timestampInSeconds;
+      if (differenceInSeconds <= maxClampToleranceInSeconds) {
+        adjustedTimestamp = startTimestampInSeconds;
+      } else {
+        return Promise.reject(new Error(`Timestamp is before start timestamp (requested: ${timestampInSeconds}sec, start: ${startTimestampInSeconds}sec, difference: ${differenceInSeconds.toFixed(3)}sec exceeds tolerance of ${maxClampToleranceInSeconds}sec)`));
+      }
     }
-    if (timestampInSeconds > endTimestampInSeconds) {
-      return Promise.reject(new Error(`Timestamp is after end timestamp (requested: ${timestampInSeconds}sec, end: ${endTimestampInSeconds})`));
+    if (roundTo4Digits(adjustedTimestamp) > roundTo4Digits(endTimestampInSeconds)) {
+      return Promise.reject(new Error(`Timestamp is after end timestamp (requested: ${timestampInSeconds}sec, end: ${endTimestampInSeconds}sec)`));
     }
-    await ensureEnoughFramesForTimestamp(timestampInSeconds);
+    await ensureEnoughFramesForTimestamp(adjustedTimestamp);
     for (let i = frameTimestamps.length - 1;i >= 0; i--) {
       const sample = frames[frameTimestamps[i]];
       if (!sample) {
         return null;
       }
-      if (roundTo4Digits(sample.timestamp) <= roundTo4Digits(timestampInSeconds) || Math.abs(sample.timestamp - timestampInSeconds) <= 0.001) {
+      if (roundTo4Digits(sample.timestamp) <= roundTo4Digits(adjustedTimestamp) || Math.abs(sample.timestamp - adjustedTimestamp) <= 0.001) {
         return sample;
       }
     }
@@ -2588,7 +2590,7 @@ var makeKeyframeManager = () => {
   }) => {
     const startPacket = await packetSink.getKeyPacket(timestamp, {
       verifyKeyPackets: true
-    });
+    }) ?? await packetSink.getFirstPacket({ verifyKeyPackets: true });
     const hasAlpha = startPacket?.sideData.alpha;
     if (hasAlpha && !canBrowserUseWebGl2()) {
       return "has-alpha";
@@ -2890,6 +2892,7 @@ var extractAudioInternal = async ({
   const durationInSeconds = durationNotYetApplyingPlaybackRate * playbackRate;
   const samples = await sampleIterator.getSamples(timeInSeconds, durationInSeconds);
   audioManager.logOpenFrames();
+  const trimStartToleranceInSeconds = 0.002;
   const audioDataArray = [];
   for (let i = 0;i < samples.length; i++) {
     const sample = samples[i];
@@ -2906,7 +2909,7 @@ var extractAudioInternal = async ({
     let trimEndInSeconds = 0;
     if (isFirstSample) {
       trimStartInSeconds = timeInSeconds - sample.timestamp;
-      if (trimStartInSeconds < 0 && trimStartInSeconds > -0.0000000001) {
+      if (trimStartInSeconds < 0 && trimStartInSeconds > -trimStartToleranceInSeconds) {
         trimStartInSeconds = 0;
       }
       if (trimStartInSeconds < 0) {
@@ -3583,7 +3586,7 @@ var VideoForPreviewAssertedShowing = ({
   const [mediaPlayerReady, setMediaPlayerReady] = useState4(false);
   const [shouldFallbackToNativeVideo, setShouldFallbackToNativeVideo] = useState4(false);
   const [playing] = Timeline2.usePlayingState();
-  const timelineContext = useContext4(Timeline2.TimelineContext);
+  const timelineContext = useContext4(Internals15.TimelineContext);
   const globalPlaybackRate = timelineContext.playbackRate;
   const sharedAudioContext = useContext4(SharedAudioContext2);
   const buffer = useBufferState2();

package/dist/media-player.js

@@ -228,12 +228,6 @@ export class MediaPlayer {
         if (currentPlaybackTime === newTime) {
             return;
         }
-        const newAudioSyncAnchor = this.sharedAudioContext.currentTime -
-            newTime / (this.playbackRate * this.globalPlaybackRate);
-        const diff = Math.abs(newAudioSyncAnchor - this.audioSyncAnchor);
-        if (diff > 0.04) {
-            this.setPlaybackTime(newTime, this.playbackRate * this.globalPlaybackRate);
-        }
         await this.videoIteratorManager?.seek({
             newTime,
             nonce,

package/dist/video/video-for-preview.js

@@ -15,7 +15,7 @@ const VideoForPreviewAssertedShowing = ({ src: unpreloadedSrc, style, playbackRa
     const [mediaPlayerReady, setMediaPlayerReady] = useState(false);
     const [shouldFallbackToNativeVideo, setShouldFallbackToNativeVideo] = useState(false);
     const [playing] = Timeline.usePlayingState();
-    const timelineContext = useContext(Timeline.TimelineContext);
+    const timelineContext = useContext(Internals.TimelineContext);
     const globalPlaybackRate = timelineContext.playbackRate;
     const sharedAudioContext = useContext(SharedAudioContext);
     const buffer = useBufferState();

package/dist/video-extraction/keyframe-bank.js

@@ -68,23 +68,35 @@ export const makeKeyframeBank = ({ startTimestampInSeconds, endTimestampInSecond
     };
     const getFrameFromTimestamp = async (timestampInSeconds) => {
         lastUsed = Date.now();
-        if (timestampInSeconds < startTimestampInSeconds) {
-            return Promise.reject(new Error(`Timestamp is before start timestamp (requested: ${timestampInSeconds}sec, start: ${startTimestampInSeconds})`));
+        // Videos may start slightly after timestamp 0 due to encoding, but if the requested timestamp is too far before the bank start, something is likely wrong.
+        const maxClampToleranceInSeconds = 0.1;
+        // If the requested timestamp is before the start of this bank, clamp it to the start if within tolerance. This handles videos that don't start at timestamp 0.
+        // For example, requesting frame at 0sec when video starts at 0.04sec should return the frame at 0.04sec.
+        // Test case: https://github.com/remotion-dev/remotion/issues/5915
+        let adjustedTimestamp = timestampInSeconds;
+        if (roundTo4Digits(timestampInSeconds) <
+            roundTo4Digits(startTimestampInSeconds)) {
+            const differenceInSeconds = startTimestampInSeconds - timestampInSeconds;
+            if (differenceInSeconds <= maxClampToleranceInSeconds) {
+                adjustedTimestamp = startTimestampInSeconds;
+            }
+            else {
+                return Promise.reject(new Error(`Timestamp is before start timestamp (requested: ${timestampInSeconds}sec, start: ${startTimestampInSeconds}sec, difference: ${differenceInSeconds.toFixed(3)}sec exceeds tolerance of ${maxClampToleranceInSeconds}sec)`));
+            }
         }
-        if (timestampInSeconds > endTimestampInSeconds) {
-            return Promise.reject(new Error(`Timestamp is after end timestamp (requested: ${timestampInSeconds}sec, end: ${endTimestampInSeconds})`));
+        if (roundTo4Digits(adjustedTimestamp) > roundTo4Digits(endTimestampInSeconds)) {
+            return Promise.reject(new Error(`Timestamp is after end timestamp (requested: ${timestampInSeconds}sec, end: ${endTimestampInSeconds}sec)`));
         }
-        await ensureEnoughFramesForTimestamp(timestampInSeconds);
+        await ensureEnoughFramesForTimestamp(adjustedTimestamp);
         for (let i = frameTimestamps.length - 1; i >= 0; i--) {
             const sample = frames[frameTimestamps[i]];
             if (!sample) {
                 return null;
             }
-            if (roundTo4Digits(sample.timestamp) <=
-                roundTo4Digits(timestampInSeconds) ||
+            if (roundTo4Digits(sample.timestamp) <= roundTo4Digits(adjustedTimestamp) ||
                 // Match 0.3333333333 to 0.33355555
                 // this does not satisfy the previous condition, since one rounds up and one rounds down
-                Math.abs(sample.timestamp - timestampInSeconds) <= 0.001) {
+                Math.abs(sample.timestamp - adjustedTimestamp) <= 0.001) {
                 return sample;
             }
         }

package/dist/video-extraction/keyframe-manager.js

@@ -111,9 +111,12 @@ export const makeKeyframeManager = () => {
         await logCacheStats(logLevel);
     };
     const getKeyframeBankOrRefetch = async ({ packetSink, timestamp, videoSampleSink, src, logLevel, }) => {
-        const startPacket = await packetSink.getKeyPacket(timestamp, {
+        // Try to get the keypacket at the requested timestamp.
+        // If it returns null (timestamp is before the first keypacket), fall back to the first packet.
+        // This matches mediabunny's internal behavior and handles videos that don't start at timestamp 0.
+        const startPacket = (await packetSink.getKeyPacket(timestamp, {
             verifyKeyPackets: true,
-        });
+        })) ?? (await packetSink.getFirstPacket({ verifyKeyPackets: true }));
         const hasAlpha = startPacket?.sideData.alpha;
         if (hasAlpha && !canBrowserUseWebGl2()) {
             return 'has-alpha';

package/dist/video-extraction/rotate-frame.d.ts

@@ -0,0 +1,4 @@
+export declare const rotateFrame: ({ frame, rotation, }: {
+    frame: VideoFrame;
+    rotation: number;
+}) => Promise<ImageBitmap>;

package/dist/video-extraction/rotate-frame.js

@@ -0,0 +1,34 @@
+export const rotateFrame = async ({ frame, rotation, }) => {
+    if (rotation === 0) {
+        const directBitmap = await createImageBitmap(frame);
+        frame.close();
+        return directBitmap;
+    }
+    const width = rotation === 90 || rotation === 270
+        ? frame.displayHeight
+        : frame.displayWidth;
+    const height = rotation === 90 || rotation === 270
+        ? frame.displayWidth
+        : frame.displayHeight;
+    const canvas = new OffscreenCanvas(width, height);
+    const ctx = canvas.getContext('2d');
+    if (!ctx) {
+        throw new Error('Could not get 2d context');
+    }
+    canvas.width = width;
+    canvas.height = height;
+    if (rotation === 90) {
+        ctx.translate(width, 0);
+    }
+    else if (rotation === 180) {
+        ctx.translate(width, height);
+    }
+    else if (rotation === 270) {
+        ctx.translate(0, height);
+    }
+    ctx.rotate(rotation * (Math.PI / 180));
+    ctx.drawImage(frame, 0, 0);
+    const bitmap = await createImageBitmap(canvas);
+    frame.close();
+    return bitmap;
+};

package/dist/video-extraction/to-video-frame-fixed-rotation.d.ts

@@ -0,0 +1,14 @@
+import type { VideoSample } from 'mediabunny';
+/**
+ * Once we convert a VideoSample to a VideoFrame, we lose the rotation
+ * https://github.com/Vanilagy/mediabunny/pull/212
+ * This will be fixed in Mediabunny v2, but for now, we need to manually fix it.
+ *
+ * I'm actually wondering if your PR is actually a breaking change
+  I would say it kinda is actually
+  Because, previously only the VideoSample had rotation but the video frame you got from .toVideoFrame() was unrotated. Now, the resulting VideoFrame will be rotated, so drawing it to a canvas will behave differently. To me, this is a breaking change
+  People's old code that manually handled the rotation will break here
+  So I think this is actually a PR for v2
+  And for Remotion, you can do a temporary workaround fix by cloning the VideoFrame and overriding rotation that way, then closing the old frame, then transferring the cloned frame
+ */
+export declare const toVideoFrameFixedRotation: (videoSample: VideoSample) => VideoFrame;

package/dist/video-extraction/to-video-frame-fixed-rotation.js

@@ -0,0 +1,41 @@
+/**
+ * Once we convert a VideoSample to a VideoFrame, we lose the rotation
+ * https://github.com/Vanilagy/mediabunny/pull/212
+ * This will be fixed in Mediabunny v2, but for now, we need to manually fix it.
+ *
+ * I'm actually wondering if your PR is actually a breaking change
+  I would say it kinda is actually
+  Because, previously only the VideoSample had rotation but the video frame you got from .toVideoFrame() was unrotated. Now, the resulting VideoFrame will be rotated, so drawing it to a canvas will behave differently. To me, this is a breaking change
+  People's old code that manually handled the rotation will break here
+  So I think this is actually a PR for v2
+  And for Remotion, you can do a temporary workaround fix by cloning the VideoFrame and overriding rotation that way, then closing the old frame, then transferring the cloned frame
+ */
+export const toVideoFrameFixedRotation = (videoSample) => {
+    const frame = videoSample.toVideoFrame();
+    if (videoSample.rotation === 0) {
+        return frame;
+    }
+    const canvas = new OffscreenCanvas(width, height);
+    const ctx = canvas.getContext('2d');
+    if (!ctx) {
+        throw new Error('Could not get 2d context');
+    }
+    canvas.width = width;
+    canvas.height = height;
+    if (canvasRotationToApply === 90) {
+        ctx.translate(width, 0);
+    }
+    else if (canvasRotationToApply === 180) {
+        ctx.translate(width, height);
+    }
+    else if (canvasRotationToApply === 270) {
+        ctx.translate(0, height);
+    }
+    console.log('sample rotation', videoSample.rotation);
+    // @ts-expect-error - rotation is not a known property of VideoFrameInit
+    const fixedFrame = new VideoFrame(frame, { rotation: videoSample.rotation });
+    frame.close();
+    // @ts-expect-error - rotation is not a known property of VideoFrameInit
+    console.log('fixed frame rotation', fixedFrame.rotation);
+    return fixedFrame;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@remotion/media",
-	"version": "4.0.372",
+	"version": "4.0.374",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
 	"module": "dist/esm/index.mjs",
@@ -22,20 +22,20 @@
 	},
 	"dependencies": {
 		"mediabunny": "1.24.3",
-		"remotion": "4.0.372",
-		"webdriverio": "9.19.2"
+		"remotion": "4.0.374"
 	},
 	"peerDependencies": {
 		"react": ">=16.8.0",
 		"react-dom": ">=16.8.0"
 	},
 	"devDependencies": {
-		"@remotion/eslint-config-internal": "4.0.372",
-		"@vitest/browser": "^3.2.4",
+		"@remotion/eslint-config-internal": "4.0.374",
+		"@vitest/browser-webdriverio": "4.0.7",
 		"eslint": "9.19.0",
 		"react": "19.0.0",
 		"react-dom": "19.0.0",
-		"vitest": "3.2.4"
+		"vitest": "4.0.7",
+		"webdriverio": "9.19.2"
 	},
 	"keywords": [],
 	"publishConfig": {