@remotion/media 4.0.373 → 4.0.375

package/dist/audio-extraction/extract-audio.js

@@ -1,6 +1,7 @@
 import { audioManager } from '../caches';
 import { combineAudioDataAndClosePrevious } from '../convert-audiodata/combine-audiodata';
-import { convertAudioData } from '../convert-audiodata/convert-audiodata';
+import { convertAudioData, fixFloatingPoint, } from '../convert-audiodata/convert-audiodata';
+import { TARGET_NUMBER_OF_CHANNELS, TARGET_SAMPLE_RATE, } from '../convert-audiodata/resample-audiodata';
 import { getSink } from '../get-sink';
 import { getTimeInSeconds } from '../get-time-in-seconds';
 const extractAudioInternal = async ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds: durationNotYetApplyingPlaybackRate, logLevel, loop, playbackRate, audioStreamIndex, trimBefore, trimAfter, fps, }) => {
@@ -44,7 +45,6 @@ const extractAudioInternal = async ({ src, timeInSeconds: unloopedTimeInSeconds,
     const durationInSeconds = durationNotYetApplyingPlaybackRate * playbackRate;
     const samples = await sampleIterator.getSamples(timeInSeconds, durationInSeconds);
     audioManager.logOpenFrames();
-    const trimStartToleranceInSeconds = 0.002;
     const audioDataArray = [];
     for (let i = 0; i < samples.length; i++) {
         const sample = samples[i];
@@ -64,14 +64,18 @@ const extractAudioInternal = async ({ src, timeInSeconds: unloopedTimeInSeconds,
         // amount of samples to shave from start and end
         let trimStartInSeconds = 0;
         let trimEndInSeconds = 0;
+        let leadingSilence = null;
         if (isFirstSample) {
-            trimStartInSeconds = timeInSeconds - sample.timestamp;
-            if (trimStartInSeconds < 0 &&
-                trimStartInSeconds > -trimStartToleranceInSeconds) {
-                trimStartInSeconds = 0;
-            }
+            trimStartInSeconds = fixFloatingPoint(timeInSeconds - sample.timestamp);
             if (trimStartInSeconds < 0) {
-                throw new Error(`trimStartInSeconds is negative: ${trimStartInSeconds}. ${JSON.stringify({ timeInSeconds, ts: sample.timestamp, d: sample.duration, isFirstSample, isLastSample, durationInSeconds, i, st: samples.map((s) => s.timestamp) })}`);
+                const silenceFrames = Math.ceil(fixFloatingPoint(-trimStartInSeconds * TARGET_SAMPLE_RATE));
+                leadingSilence = {
+                    data: new Int16Array(silenceFrames * TARGET_NUMBER_OF_CHANNELS),
+                    numberOfFrames: silenceFrames,
+                    timestamp: timeInSeconds * 1000000,
+                    durationInMicroSeconds: (silenceFrames / TARGET_SAMPLE_RATE) * 1000000,
+                };
+                trimStartInSeconds = 0;
             }
         }
         if (isLastSample) {
@@ -93,6 +97,9 @@ const extractAudioInternal = async ({ src, timeInSeconds: unloopedTimeInSeconds,
         if (audioData.numberOfFrames === 0) {
             continue;
         }
+        if (leadingSilence) {
+            audioDataArray.push(leadingSilence);
+        }
         audioDataArray.push(audioData);
     }
     if (audioDataArray.length === 0) {

package/dist/convert-audiodata/convert-audiodata.js

@@ -1,11 +1,12 @@
 import { resampleAudioData, TARGET_NUMBER_OF_CHANNELS, TARGET_SAMPLE_RATE, } from './resample-audiodata';
 const FORMAT = 's16';
 export const fixFloatingPoint = (value) => {
-    if (value % 1 < 0.0000001) {
-        return Math.floor(value);
+    const decimal = Math.abs(value % 1);
+    if (decimal < 0.0000001) {
+        return value < 0 ? Math.ceil(value) : Math.floor(value);
     }
-    if (value % 1 > 0.9999999) {
-        return Math.ceil(value);
+    if (decimal > 0.9999999) {
+        return value < 0 ? Math.floor(value) : Math.ceil(value);
     }
     return value;
 };

package/dist/esm/index.mjs

@@ -994,11 +994,6 @@ class MediaPlayer {
     if (currentPlaybackTime === newTime) {
       return;
     }
-    const newAudioSyncAnchor = this.sharedAudioContext.currentTime - newTime / (this.playbackRate * this.globalPlaybackRate);
-    const diff = Math.abs(newAudioSyncAnchor - this.audioSyncAnchor);
-    if (diff > 0.04) {
-      this.setPlaybackTime(newTime, this.playbackRate * this.globalPlaybackRate);
-    }
     await this.videoIteratorManager?.seek({
       newTime,
       nonce
@@ -2738,11 +2733,12 @@ var getMaxVideoCacheSize = (logLevel) => {
 // src/convert-audiodata/convert-audiodata.ts
 var FORMAT = "s16";
 var fixFloatingPoint2 = (value) => {
-  if (value % 1 < 0.0000001) {
-    return Math.floor(value);
+  const decimal = Math.abs(value % 1);
+  if (decimal < 0.0000001) {
+    return value < 0 ? Math.ceil(value) : Math.floor(value);
   }
-  if (value % 1 > 0.9999999) {
-    return Math.ceil(value);
+  if (decimal > 0.9999999) {
+    return value < 0 ? Math.floor(value) : Math.ceil(value);
   }
   return value;
 };
@@ -2897,7 +2893,6 @@ var extractAudioInternal = async ({
   const durationInSeconds = durationNotYetApplyingPlaybackRate * playbackRate;
   const samples = await sampleIterator.getSamples(timeInSeconds, durationInSeconds);
   audioManager.logOpenFrames();
-  const trimStartToleranceInSeconds = 0.002;
   const audioDataArray = [];
   for (let i = 0;i < samples.length; i++) {
     const sample = samples[i];
@@ -2912,13 +2907,18 @@ var extractAudioInternal = async ({
     const audioDataRaw = sample.toAudioData();
     let trimStartInSeconds = 0;
     let trimEndInSeconds = 0;
+    let leadingSilence = null;
     if (isFirstSample) {
-      trimStartInSeconds = timeInSeconds - sample.timestamp;
-      if (trimStartInSeconds < 0 && trimStartInSeconds > -trimStartToleranceInSeconds) {
-        trimStartInSeconds = 0;
-      }
+      trimStartInSeconds = fixFloatingPoint2(timeInSeconds - sample.timestamp);
       if (trimStartInSeconds < 0) {
-        throw new Error(`trimStartInSeconds is negative: ${trimStartInSeconds}. ${JSON.stringify({ timeInSeconds, ts: sample.timestamp, d: sample.duration, isFirstSample, isLastSample, durationInSeconds, i, st: samples.map((s) => s.timestamp) })}`);
+        const silenceFrames = Math.ceil(fixFloatingPoint2(-trimStartInSeconds * TARGET_SAMPLE_RATE));
+        leadingSilence = {
+          data: new Int16Array(silenceFrames * TARGET_NUMBER_OF_CHANNELS),
+          numberOfFrames: silenceFrames,
+          timestamp: timeInSeconds * 1e6,
+          durationInMicroSeconds: silenceFrames / TARGET_SAMPLE_RATE * 1e6
+        };
+        trimStartInSeconds = 0;
       }
     }
     if (isLastSample) {
@@ -2936,6 +2936,9 @@ var extractAudioInternal = async ({
     if (audioData.numberOfFrames === 0) {
       continue;
     }
+    if (leadingSilence) {
+      audioDataArray.push(leadingSilence);
+    }
     audioDataArray.push(audioData);
   }
   if (audioDataArray.length === 0) {
@@ -3023,6 +3026,39 @@ var extractFrame = (params) => {
   return queue2;
 };
 
+// src/video-extraction/rotate-frame.ts
+var rotateFrame = async ({
+  frame,
+  rotation
+}) => {
+  if (rotation === 0) {
+    const directBitmap = await createImageBitmap(frame);
+    frame.close();
+    return directBitmap;
+  }
+  const width = rotation === 90 || rotation === 270 ? frame.displayHeight : frame.displayWidth;
+  const height = rotation === 90 || rotation === 270 ? frame.displayWidth : frame.displayHeight;
+  const canvas = new OffscreenCanvas(width, height);
+  const ctx = canvas.getContext("2d");
+  if (!ctx) {
+    throw new Error("Could not get 2d context");
+  }
+  canvas.width = width;
+  canvas.height = height;
+  if (rotation === 90) {
+    ctx.translate(width, 0);
+  } else if (rotation === 180) {
+    ctx.translate(width, height);
+  } else if (rotation === 270) {
+    ctx.translate(0, height);
+  }
+  ctx.rotate(rotation * (Math.PI / 180));
+  ctx.drawImage(frame, 0, 0);
+  const bitmap = await createImageBitmap(canvas);
+  frame.close();
+  return bitmap;
+};
+
 // src/extract-frame-and-audio.ts
 var extractFrameAndAudio = async ({
   src,
@@ -3093,9 +3129,20 @@ var extractFrameAndAudio = async ({
         durationInSeconds: frame?.type === "success" ? frame.durationInSeconds : null
       };
     }
+    if (!frame?.frame) {
+      return {
+        type: "success",
+        frame: null,
+        audio: audio?.data ?? null,
+        durationInSeconds: audio?.durationInSeconds ?? null
+      };
+    }
     return {
       type: "success",
-      frame: frame?.frame?.toVideoFrame() ?? null,
+      frame: await rotateFrame({
+        frame: frame.frame.toVideoFrame(),
+        rotation: frame.frame.rotation
+      }),
       audio: audio?.data ?? null,
       durationInSeconds: audio?.durationInSeconds ?? null
     };
@@ -3163,10 +3210,9 @@ if (typeof window !== "undefined" && window.remotion_broadcastChannel && window.
           return;
         }
         const { frame, audio, durationInSeconds } = result;
-        const videoFrame = frame;
-        const imageBitmap = videoFrame ? await createImageBitmap(videoFrame) : null;
-        if (videoFrame) {
-          videoFrame.close();
+        const imageBitmap = frame ? await createImageBitmap(frame) : null;
+        if (frame) {
+          frame.close();
         }
         const response = {
           type: "response-success",
@@ -3176,7 +3222,6 @@ if (typeof window !== "undefined" && window.remotion_broadcastChannel && window.
           durationInSeconds: durationInSeconds ?? null
         };
         window.remotion_broadcastChannel.postMessage(response);
-        videoFrame?.close();
       } catch (error) {
         const response = {
           type: "response-error",
@@ -4069,8 +4114,8 @@ var VideoForRendering = ({
         if (!context) {
           return;
         }
-        context.canvas.width = imageBitmap instanceof ImageBitmap ? imageBitmap.width : imageBitmap.displayWidth;
-        context.canvas.height = imageBitmap instanceof ImageBitmap ? imageBitmap.height : imageBitmap.displayHeight;
+        context.canvas.width = imageBitmap.width;
+        context.canvas.height = imageBitmap.height;
         context.canvas.style.aspectRatio = `${context.canvas.width} / ${context.canvas.height}`;
         context.drawImage(imageBitmap, 0, 0);
         imageBitmap.close();

package/dist/extract-frame-and-audio.js

@@ -1,6 +1,7 @@
 import { extractAudio } from './audio-extraction/extract-audio';
 import { isNetworkError } from './is-network-error';
 import { extractFrame } from './video-extraction/extract-frame';
+import { rotateFrame } from './video-extraction/rotate-frame';
 export const extractFrameAndAudio = async ({ src, timeInSeconds, logLevel, durationInSeconds, playbackRate, includeAudio, includeVideo, loop, audioStreamIndex, trimAfter, trimBefore, fps, }) => {
     try {
         const [frame, audio] = await Promise.all([
@@ -61,9 +62,20 @@ export const extractFrameAndAudio = async ({ src, timeInSeconds, logLevel, durat
                 durationInSeconds: frame?.type === 'success' ? frame.durationInSeconds : null,
             };
         }
+        if (!frame?.frame) {
+            return {
+                type: 'success',
+                frame: null,
+                audio: audio?.data ?? null,
+                durationInSeconds: audio?.durationInSeconds ?? null,
+            };
+        }
         return {
             type: 'success',
-            frame: frame?.frame?.toVideoFrame() ?? null,
+            frame: await rotateFrame({
+                frame: frame.frame.toVideoFrame(),
+                rotation: frame.frame.rotation,
+            }),
             audio: audio?.data ?? null,
             durationInSeconds: audio?.durationInSeconds ?? null,
         };

package/dist/media-player.js

@@ -228,12 +228,6 @@ export class MediaPlayer {
         if (currentPlaybackTime === newTime) {
             return;
         }
-        const newAudioSyncAnchor = this.sharedAudioContext.currentTime -
-            newTime / (this.playbackRate * this.globalPlaybackRate);
-        const diff = Math.abs(newAudioSyncAnchor - this.audioSyncAnchor);
-        if (diff > 0.04) {
-            this.setPlaybackTime(newTime, this.playbackRate * this.globalPlaybackRate);
-        }
         await this.videoIteratorManager?.seek({
             newTime,
             nonce,

package/dist/video/video-for-rendering.js

@@ -120,14 +120,8 @@ export const VideoForRendering = ({ volume: volumeProp, playbackRate, src, muted
                 if (!context) {
                     return;
                 }
-                context.canvas.width =
-                    imageBitmap instanceof ImageBitmap
-                        ? imageBitmap.width
-                        : imageBitmap.displayWidth;
-                context.canvas.height =
-                    imageBitmap instanceof ImageBitmap
-                        ? imageBitmap.height
-                        : imageBitmap.displayHeight;
+                context.canvas.width = imageBitmap.width;
+                context.canvas.height = imageBitmap.height;
                 context.canvas.style.aspectRatio = `${context.canvas.width} / ${context.canvas.height}`;
                 context.drawImage(imageBitmap, 0, 0);
                 imageBitmap.close();

package/dist/video-extraction/extract-frame-via-broadcast-channel.d.ts

@@ -2,7 +2,7 @@ import { type LogLevel } from 'remotion';
 import type { PcmS16AudioData } from '../convert-audiodata/convert-audiodata';
 export type ExtractFrameViaBroadcastChannelResult = {
     type: 'success';
-    frame: ImageBitmap | VideoFrame | null;
+    frame: ImageBitmap | null;
     audio: PcmS16AudioData | null;
     durationInSeconds: number | null;
 } | {

package/dist/video-extraction/extract-frame-via-broadcast-channel.js

@@ -56,12 +56,9 @@ if (typeof window !== 'undefined' &&
                     return;
                 }
                 const { frame, audio, durationInSeconds } = result;
-                const videoFrame = frame;
-                const imageBitmap = videoFrame
-                    ? await createImageBitmap(videoFrame)
-                    : null;
-                if (videoFrame) {
-                    videoFrame.close();
+                const imageBitmap = frame ? await createImageBitmap(frame) : null;
+                if (frame) {
+                    frame.close();
                 }
                 const response = {
                     type: 'response-success',
@@ -71,7 +68,6 @@ if (typeof window !== 'undefined' &&
                     durationInSeconds: durationInSeconds ?? null,
                 };
                 window.remotion_broadcastChannel.postMessage(response);
-                videoFrame?.close();
             }
             catch (error) {
                 const response = {

package/dist/video-extraction/rotate-frame.d.ts

@@ -0,0 +1,4 @@
+export declare const rotateFrame: ({ frame, rotation, }: {
+    frame: VideoFrame;
+    rotation: number;
+}) => Promise<ImageBitmap>;

package/dist/video-extraction/rotate-frame.js

@@ -0,0 +1,34 @@
+export const rotateFrame = async ({ frame, rotation, }) => {
+    if (rotation === 0) {
+        const directBitmap = await createImageBitmap(frame);
+        frame.close();
+        return directBitmap;
+    }
+    const width = rotation === 90 || rotation === 270
+        ? frame.displayHeight
+        : frame.displayWidth;
+    const height = rotation === 90 || rotation === 270
+        ? frame.displayWidth
+        : frame.displayHeight;
+    const canvas = new OffscreenCanvas(width, height);
+    const ctx = canvas.getContext('2d');
+    if (!ctx) {
+        throw new Error('Could not get 2d context');
+    }
+    canvas.width = width;
+    canvas.height = height;
+    if (rotation === 90) {
+        ctx.translate(width, 0);
+    }
+    else if (rotation === 180) {
+        ctx.translate(width, height);
+    }
+    else if (rotation === 270) {
+        ctx.translate(0, height);
+    }
+    ctx.rotate(rotation * (Math.PI / 180));
+    ctx.drawImage(frame, 0, 0);
+    const bitmap = await createImageBitmap(canvas);
+    frame.close();
+    return bitmap;
+};

package/dist/video-extraction/to-video-frame-fixed-rotation.d.ts

@@ -0,0 +1,14 @@
+import type { VideoSample } from 'mediabunny';
+/**
+ * Once we convert a VideoSample to a VideoFrame, we lose the rotation
+ * https://github.com/Vanilagy/mediabunny/pull/212
+ * This will be fixed in Mediabunny v2, but for now, we need to manually fix it.
+ *
+ * I'm actually wondering if your PR is actually a breaking change
+  I would say it kinda is actually
+  Because, previously only the VideoSample had rotation but the video frame you got from .toVideoFrame() was unrotated. Now, the resulting VideoFrame will be rotated, so drawing it to a canvas will behave differently. To me, this is a breaking change
+  People's old code that manually handled the rotation will break here
+  So I think this is actually a PR for v2
+  And for Remotion, you can do a temporary workaround fix by cloning the VideoFrame and overriding rotation that way, then closing the old frame, then transferring the cloned frame
+ */
+export declare const toVideoFrameFixedRotation: (videoSample: VideoSample) => VideoFrame;

package/dist/video-extraction/to-video-frame-fixed-rotation.js

@@ -0,0 +1,41 @@
+/**
+ * Once we convert a VideoSample to a VideoFrame, we lose the rotation
+ * https://github.com/Vanilagy/mediabunny/pull/212
+ * This will be fixed in Mediabunny v2, but for now, we need to manually fix it.
+ *
+ * I'm actually wondering if your PR is actually a breaking change
+  I would say it kinda is actually
+  Because, previously only the VideoSample had rotation but the video frame you got from .toVideoFrame() was unrotated. Now, the resulting VideoFrame will be rotated, so drawing it to a canvas will behave differently. To me, this is a breaking change
+  People's old code that manually handled the rotation will break here
+  So I think this is actually a PR for v2
+  And for Remotion, you can do a temporary workaround fix by cloning the VideoFrame and overriding rotation that way, then closing the old frame, then transferring the cloned frame
+ */
+export const toVideoFrameFixedRotation = (videoSample) => {
+    const frame = videoSample.toVideoFrame();
+    if (videoSample.rotation === 0) {
+        return frame;
+    }
+    const canvas = new OffscreenCanvas(width, height);
+    const ctx = canvas.getContext('2d');
+    if (!ctx) {
+        throw new Error('Could not get 2d context');
+    }
+    canvas.width = width;
+    canvas.height = height;
+    if (canvasRotationToApply === 90) {
+        ctx.translate(width, 0);
+    }
+    else if (canvasRotationToApply === 180) {
+        ctx.translate(width, height);
+    }
+    else if (canvasRotationToApply === 270) {
+        ctx.translate(0, height);
+    }
+    console.log('sample rotation', videoSample.rotation);
+    // @ts-expect-error - rotation is not a known property of VideoFrameInit
+    const fixedFrame = new VideoFrame(frame, { rotation: videoSample.rotation });
+    frame.close();
+    // @ts-expect-error - rotation is not a known property of VideoFrameInit
+    console.log('fixed frame rotation', fixedFrame.rotation);
+    return fixedFrame;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@remotion/media",
-	"version": "4.0.373",
+	"version": "4.0.375",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
 	"module": "dist/esm/index.mjs",
@@ -21,15 +21,15 @@
 		"make": "tsc -d && bun --env-file=../.env.bundle bundle.ts"
 	},
 	"dependencies": {
-		"mediabunny": "1.24.3",
-		"remotion": "4.0.373"
+		"mediabunny": "1.24.5",
+		"remotion": "4.0.375"
 	},
 	"peerDependencies": {
 		"react": ">=16.8.0",
 		"react-dom": ">=16.8.0"
 	},
 	"devDependencies": {
-		"@remotion/eslint-config-internal": "4.0.373",
+		"@remotion/eslint-config-internal": "4.0.375",
 		"@vitest/browser-webdriverio": "4.0.7",
 		"eslint": "9.19.0",
 		"react": "19.0.0",