@remotion/webcodecs 4.0.267 → 4.0.268

package/dist/create/matroska/matroska-utils.d.ts

@@ -19,7 +19,7 @@ export type EbmlParsedOrUint8Array<T extends Ebml> = {
     value: EbmlValueOrUint8Array<T>;
     minVintWidth: number | null;
 };
-export declare const measureEBMLVarInt: (value: number) => 2 | 1 | 4 | 3 | 5 | 6;
+export declare const measureEBMLVarInt: (value: number) => 2 | 1 | 5 | 4 | 3 | 6;
 export declare const getVariableInt: (value: number, minWidth: number | null) => Uint8Array;
 export declare const makeMatroskaBytes: (fields: PossibleEbmlOrUint8Array) => BytesAndOffset;
 export type PossibleEbmlOrUint8Array = Prettify<{

package/dist/esm/index.mjs

@@ -1954,6 +1954,59 @@ var onFrame = async ({
   }
 };
 
+// src/sort-video-frames.ts
+var MAX_QUEUE_SIZE = 5;
+var videoFrameSorter = ({
+  onRelease,
+  controller
+}) => {
+  const frames = [];
+  const releaseFrame = async () => {
+    await controller._internals.checkForAbortAndPause();
+    const frame = frames.shift();
+    if (frame) {
+      await onRelease(frame);
+    }
+  };
+  const sortFrames = () => {
+    frames.sort((a, b) => a.timestamp - b.timestamp);
+  };
+  const releaseIfQueueFull = async () => {
+    if (frames.length >= MAX_QUEUE_SIZE) {
+      sortFrames();
+      await releaseFrame();
+    }
+  };
+  const addFrame = (frame) => {
+    frames.push(frame);
+  };
+  const inputFrame = async (frame) => {
+    addFrame(frame);
+    await releaseIfQueueFull();
+  };
+  const onAbort = () => {
+    while (frames.length > 0) {
+      const frame = frames.shift();
+      if (frame) {
+        frame.close();
+      }
+    }
+    frames.length = 0;
+  };
+  const flush = async () => {
+    sortFrames();
+    while (frames.length > 0) {
+      await releaseFrame();
+    }
+    controller._internals.signal.removeEventListener("abort", onAbort);
+  };
+  controller._internals.signal.addEventListener("abort", onAbort);
+  return {
+    inputFrame,
+    flush
+  };
+};
+
 // src/video-decoder.ts
 var createVideoDecoder = ({
   onFrame: onFrame2,
@@ -1969,28 +2022,38 @@ var createVideoDecoder = ({
     progress
   });
   let outputQueue = Promise.resolve();
+  const addToQueue = (frame) => {
+    const cleanup = () => {
+      frame.close();
+    };
+    controller._internals.signal.addEventListener("abort", cleanup, {
+      once: true
+    });
+    outputQueue = outputQueue.then(() => {
+      if (controller._internals.signal.aborted) {
+        return;
+      }
+      return onFrame2(frame);
+    }).then(() => {
+      ioSynchronizer.onProcessed();
+    }).catch((err) => {
+      onError(err);
+    }).finally(() => {
+      controller._internals.signal.removeEventListener("abort", cleanup);
+      cleanup();
+    });
+    return outputQueue;
+  };
+  const frameSorter = videoFrameSorter({
+    controller,
+    onRelease: async (frame) => {
+      await addToQueue(frame);
+    }
+  });
   const videoDecoder = new VideoDecoder({
-    output(inputFrame) {
-      ioSynchronizer.onOutput(inputFrame.timestamp);
-      const abortHandler = () => {
-        inputFrame.close();
-      };
-      controller._internals.signal.addEventListener("abort", abortHandler, {
-        once: true
-      });
-      outputQueue = outputQueue.then(() => {
-        if (controller._internals.signal.aborted) {
-          return;
-        }
-        return onFrame2(inputFrame);
-      }).then(() => {
-        ioSynchronizer.onProcessed();
-        controller._internals.signal.removeEventListener("abort", abortHandler);
-        return Promise.resolve();
-      }).catch((err) => {
-        inputFrame.close();
-        onError(err);
-      });
+    output(frame) {
+      ioSynchronizer.onOutput(frame.timestamp);
+      frameSorter.inputFrame(frame);
     },
     error(error) {
       onError(error);
@@ -2018,13 +2081,10 @@ var createVideoDecoder = ({
     progress.setPossibleLowestTimestamp(Math.min(sample.timestamp, sample.dts ?? Infinity, sample.cts ?? Infinity));
     await ioSynchronizer.waitFor({
       unemitted: 20,
-      unprocessed: 2,
+      unprocessed: 10,
       minimumProgress: sample.timestamp - 1e7,
       controller
     });
-    if (sample.type === "key") {
-      await videoDecoder.flush();
-    }
     videoDecoder.decode(new EncodedVideoChunk(sample));
     ioSynchronizer.inputItem(sample.timestamp, sample.type === "key");
   };
@@ -2037,6 +2097,8 @@ var createVideoDecoder = ({
     waitForFinish: async () => {
       await videoDecoder.flush();
       Log.verbose(logLevel, "Flushed video decoder");
+      await frameSorter.flush();
+      Log.verbose(logLevel, "Frame sorter flushed");
       await ioSynchronizer.waitForFinish(controller);
       Log.verbose(logLevel, "IO synchro finished");
       await outputQueue;

package/dist/sort-video-frames.d.ts

@@ -0,0 +1,8 @@
+import type { WebCodecsController } from './webcodecs-controller';
+export declare const videoFrameSorter: ({ onRelease, controller, }: {
+    onRelease: (frame: VideoFrame) => Promise<void>;
+    controller: WebCodecsController;
+}) => {
+    inputFrame: (frame: VideoFrame) => Promise<void>;
+    flush: () => Promise<void>;
+};

package/dist/sort-video-frames.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.videoFrameSorter = void 0;
+const MAX_QUEUE_SIZE = 5;
+const videoFrameSorter = ({ onRelease, controller, }) => {
+    const frames = [];
+    const releaseFrame = async () => {
+        await controller._internals.checkForAbortAndPause();
+        const frame = frames.shift();
+        if (frame) {
+            await onRelease(frame);
+        }
+    };
+    const sortFrames = () => {
+        frames.sort((a, b) => a.timestamp - b.timestamp);
+    };
+    const releaseIfQueueFull = async () => {
+        if (frames.length >= MAX_QUEUE_SIZE) {
+            sortFrames();
+            await releaseFrame();
+        }
+    };
+    const addFrame = (frame) => {
+        frames.push(frame);
+    };
+    const inputFrame = async (frame) => {
+        addFrame(frame);
+        await releaseIfQueueFull();
+    };
+    const onAbort = () => {
+        while (frames.length > 0) {
+            const frame = frames.shift();
+            if (frame) {
+                frame.close();
+            }
+        }
+        frames.length = 0;
+    };
+    const flush = async () => {
+        sortFrames();
+        while (frames.length > 0) {
+            await releaseFrame();
+        }
+        controller._internals.signal.removeEventListener('abort', onAbort);
+    };
+    controller._internals.signal.addEventListener('abort', onAbort);
+    return {
+        inputFrame,
+        flush,
+    };
+};
+exports.videoFrameSorter = videoFrameSorter;

package/dist/video-decoder.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createVideoDecoder = void 0;
 const io_synchronizer_1 = require("./io-manager/io-synchronizer");
 const log_1 = require("./log");
+const sort_video_frames_1 = require("./sort-video-frames");
 const createVideoDecoder = ({ onFrame, onError, controller, config, logLevel, progress, }) => {
     const ioSynchronizer = (0, io_synchronizer_1.makeIoSynchronizer)({
         logLevel,
@@ -10,31 +11,42 @@ const createVideoDecoder = ({ onFrame, onError, controller, config, logLevel, pr
         progress,
     });
     let outputQueue = Promise.resolve();
+    const addToQueue = (frame) => {
+        const cleanup = () => {
+            frame.close();
+        };
+        controller._internals.signal.addEventListener('abort', cleanup, {
+            once: true,
+        });
+        outputQueue = outputQueue
+            .then(() => {
+            if (controller._internals.signal.aborted) {
+                return;
+            }
+            return onFrame(frame);
+        })
+            .then(() => {
+            ioSynchronizer.onProcessed();
+        })
+            .catch((err) => {
+            onError(err);
+        })
+            .finally(() => {
+            controller._internals.signal.removeEventListener('abort', cleanup);
+            cleanup();
+        });
+        return outputQueue;
+    };
+    const frameSorter = (0, sort_video_frames_1.videoFrameSorter)({
+        controller,
+        onRelease: async (frame) => {
+            await addToQueue(frame);
+        },
+    });
     const videoDecoder = new VideoDecoder({
-        output(inputFrame) {
-            ioSynchronizer.onOutput(inputFrame.timestamp);
-            const abortHandler = () => {
-                inputFrame.close();
-            };
-            controller._internals.signal.addEventListener('abort', abortHandler, {
-                once: true,
-            });
-            outputQueue = outputQueue
-                .then(() => {
-                if (controller._internals.signal.aborted) {
-                    return;
-                }
-                return onFrame(inputFrame);
-            })
-                .then(() => {
-                ioSynchronizer.onProcessed();
-                controller._internals.signal.removeEventListener('abort', abortHandler);
-                return Promise.resolve();
-            })
-                .catch((err) => {
-                inputFrame.close();
-                onError(err);
-            });
+        output(frame) {
+            ioSynchronizer.onOutput(frame.timestamp);
+            frameSorter.inputFrame(frame);
         },
         error(error) {
             onError(error);
@@ -65,13 +77,14 @@ const createVideoDecoder = ({ onFrame, onError, controller, config, logLevel, pr
         progress.setPossibleLowestTimestamp(Math.min(sample.timestamp, (_a = sample.dts) !== null && _a !== void 0 ? _a : Infinity, (_b = sample.cts) !== null && _b !== void 0 ? _b : Infinity));
         await ioSynchronizer.waitFor({
             unemitted: 20,
-            unprocessed: 2,
+            unprocessed: 10,
             minimumProgress: sample.timestamp - 10000000,
             controller,
         });
-        if (sample.type === 'key') {
-            await videoDecoder.flush();
-        }
+        // Don't flush here.
+        // We manually keep track of the memory with the IO synchornizer.
+        // Example of flushing breaking things:
+        // IMG_2310.MOV has B-frames, and if we flush on a keyframe, we discard some frames that are yet to come.
         videoDecoder.decode(new EncodedVideoChunk(sample));
         ioSynchronizer.inputItem(sample.timestamp, sample.type === 'key');
     };
@@ -84,6 +97,8 @@ const createVideoDecoder = ({ onFrame, onError, controller, config, logLevel, pr
         waitForFinish: async () => {
             await videoDecoder.flush();
             log_1.Log.verbose(logLevel, 'Flushed video decoder');
+            await frameSorter.flush();
+            log_1.Log.verbose(logLevel, 'Frame sorter flushed');
             await ioSynchronizer.waitForFinish(controller);
             log_1.Log.verbose(logLevel, 'IO synchro finished');
             await outputQueue;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remotion/webcodecs",
-  "version": "4.0.267",
+  "version": "4.0.268",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "module": "dist/esm/index.mjs",
@@ -17,15 +17,15 @@
   "author": "Jonny Burger <jonny@remotion.dev>",
   "license": "Remotion License (See https://remotion.dev/docs/webcodecs#license)",
   "dependencies": {
-    "@remotion/media-parser": "4.0.267",
-    "@remotion/licensing": "4.0.267"
+    "@remotion/media-parser": "4.0.268",
+    "@remotion/licensing": "4.0.268"
   },
   "peerDependencies": {},
   "devDependencies": {
     "@types/dom-webcodecs": "0.1.11",
     "eslint": "9.19.0",
-    "@remotion/example-videos": "4.0.267",
-    "@remotion/eslint-config-internal": "4.0.267"
+    "@remotion/eslint-config-internal": "4.0.268",
+    "@remotion/example-videos": "4.0.268"
   },
   "keywords": [],
   "publishConfig": {