@remotion/webcodecs 4.0.344 → 4.0.345

package/dist/audio-decoder.d.ts

@@ -0,0 +1,24 @@
+import type { MediaParserLogLevel } from '@remotion/media-parser';
+import type { WebCodecsController } from './webcodecs-controller';
+export type WebCodecsAudioDecoder = {
+    decode: (audioSample: EncodedAudioChunkInit | EncodedAudioChunk) => void;
+    close: () => void;
+    flush: () => Promise<void>;
+    waitForFinish: () => Promise<void>;
+    waitForQueueToBeLessThan: (items: number) => Promise<void>;
+};
+export type CreateAudioDecoderInit = {
+    onFrame: (frame: AudioData) => Promise<void> | void;
+    onError: (error: Error) => void;
+    controller: WebCodecsController | null;
+    config: AudioDecoderConfig;
+    logLevel: MediaParserLogLevel;
+};
+export declare const internalCreateAudioDecoder: ({ onFrame, onError, controller, config, logLevel, }: CreateAudioDecoderInit) => WebCodecsAudioDecoder;
+export declare const createAudioDecoder: ({ onFrame, onError, controller, track, logLevel, }: {
+    track: AudioDecoderConfig;
+    onFrame: (frame: AudioData) => Promise<void> | void;
+    onError: (error: Error) => void;
+    controller?: WebCodecsController | null;
+    logLevel?: MediaParserLogLevel;
+}) => WebCodecsAudioDecoder;

package/dist/audio-decoder.js

@@ -0,0 +1,106 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createAudioDecoder = exports.internalCreateAudioDecoder = void 0;
+const get_wave_audio_decoder_1 = require("./get-wave-audio-decoder");
+const io_synchronizer_1 = require("./io-manager/io-synchronizer");
+const internalCreateAudioDecoder = ({ onFrame, onError, controller, config, logLevel, }) => {
+    if (controller &&
+        controller._internals._mediaParserController._internals.signal.aborted) {
+        throw new Error('Not creating audio decoder, already aborted');
+    }
+    const ioSynchronizer = (0, io_synchronizer_1.makeIoSynchronizer)({
+        logLevel,
+        label: 'Audio decoder',
+        controller,
+    });
+    if (config.codec === 'pcm-s16') {
+        return (0, get_wave_audio_decoder_1.getWaveAudioDecoder)({
+            onFrame,
+            config,
+            sampleFormat: 's16',
+            logLevel,
+            ioSynchronizer,
+            onError,
+        });
+    }
+    const audioDecoder = new AudioDecoder({
+        async output(frame) {
+            try {
+                await onFrame(frame);
+            }
+            catch (err) {
+                frame.close();
+                onError(err);
+            }
+            ioSynchronizer.onOutput(frame.timestamp + (frame.duration ?? 0));
+        },
+        error(error) {
+            onError(error);
+        },
+    });
+    const close = () => {
+        if (controller) {
+            controller._internals._mediaParserController._internals.signal.removeEventListener('abort', 
+            // eslint-disable-next-line @typescript-eslint/no-use-before-define
+            onAbort);
+        }
+        if (audioDecoder.state === 'closed') {
+            return;
+        }
+        audioDecoder.close();
+    };
+    const onAbort = () => {
+        close();
+    };
+    if (controller) {
+        controller._internals._mediaParserController._internals.signal.addEventListener('abort', onAbort);
+    }
+    audioDecoder.configure(config);
+    const processSample = (audioSample) => {
+        if (audioDecoder.state === 'closed') {
+            return;
+        }
+        // Don't flush, it messes up the audio
+        const chunk = audioSample instanceof EncodedAudioChunk
+            ? audioSample
+            : new EncodedAudioChunk(audioSample);
+        audioDecoder.decode(chunk);
+        // https://test-streams.mux.dev/x36xhzz/url_0/url_525/193039199_mp4_h264_aac_hd_7.ts
+        // has a 16 byte audio sample at the end which chrome does not decode
+        // Might be empty audio
+        // For now only reporting chunks that are bigger than that
+        // 16 was chosen arbitrarily, can be improved
+        if (chunk.byteLength > 16) {
+            ioSynchronizer.inputItem(chunk.timestamp);
+        }
+    };
+    return {
+        decode: (sample) => {
+            processSample(sample);
+        },
+        waitForFinish: async () => {
+            // Firefox might throw "Needs to be configured first"
+            try {
+                await audioDecoder.flush();
+            }
+            catch { }
+            await ioSynchronizer.waitForFinish();
+        },
+        close,
+        flush: async () => {
+            await audioDecoder.flush();
+        },
+        waitForQueueToBeLessThan: ioSynchronizer.waitForQueueSize,
+    };
+};
+exports.internalCreateAudioDecoder = internalCreateAudioDecoder;
+const createAudioDecoder = ({ onFrame, onError, controller, track, logLevel, }) => {
+    return (0, exports.internalCreateAudioDecoder)({
+        onFrame,
+        onError,
+        controller: controller ?? null,
+        config: track,
+        logLevel: logLevel ?? 'error',
+    });
+};
+exports.createAudioDecoder = createAudioDecoder;

package/dist/create-audio-decoder.js

@@ -26,6 +26,16 @@ const internalCreateAudioDecoder = async ({ onFrame, onError, controller, config
             onError,
         });
     }
+    if (config.codec === 'pcm-s24') {
+        return (0, get_wave_audio_decoder_1.getWaveAudioDecoder)({
+            onFrame,
+            config,
+            sampleFormat: 's24',
+            logLevel,
+            ioSynchronizer,
+            onError,
+        });
+    }
     const audioDecoder = new AudioDecoder({
         async output(frame) {
             try {

package/dist/create-frames.d.ts

@@ -0,0 +1,8 @@
+export declare const extractFrames: ({ fromSeconds, toSeconds, width, height, src, onFrame, }: {
+    fromSeconds: number;
+    toSeconds: number;
+    width: number;
+    height: number;
+    src: string;
+    onFrame: (frame: VideoFrame) => void;
+}) => Promise<void>;

package/dist/create-frames.js

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractFrames = void 0;
+const media_parser_1 = require("@remotion/media-parser");
+const worker_1 = require("@remotion/media-parser/worker");
+const create_video_decoder_1 = require("./create-video-decoder");
+const extractFrames = async ({ fromSeconds, toSeconds, width, height, src, onFrame, }) => {
+    const controller = (0, media_parser_1.mediaParserController)();
+    controller.seek(fromSeconds);
+    const segmentDuration = toSeconds - fromSeconds;
+    const expectedFrames = [];
+    try {
+        await (0, worker_1.parseMediaOnWebWorker)({
+            src: new URL(src, window.location.href).toString(),
+            acknowledgeRemotionLicense: true,
+            controller,
+            onVideoTrack: ({ track }) => {
+                const aspectRatio = track.width / track.height;
+                const framesFitInWidth = Math.ceil(width / (height * aspectRatio));
+                const timestampTargets = [];
+                for (let i = 0; i < framesFitInWidth; i++) {
+                    timestampTargets.push(fromSeconds +
+                        ((segmentDuration * media_parser_1.WEBCODECS_TIMESCALE) / framesFitInWidth) *
+                            (i + 0.5));
+                }
+                const decoder = (0, create_video_decoder_1.createVideoDecoder)({
+                    onFrame: (frame) => {
+                        if (frame.timestamp >= expectedFrames[0] - 1) {
+                            expectedFrames.shift();
+                            onFrame(frame);
+                        }
+                        frame.close();
+                    },
+                    onError: console.error,
+                    track,
+                });
+                const queued = [];
+                return async (sample) => {
+                    const nextTimestampWeWant = timestampTargets[0];
+                    if (nextTimestampWeWant === undefined) {
+                        throw new Error('this should not happen');
+                    }
+                    if (sample.type === 'key') {
+                        queued.length = 0;
+                    }
+                    queued.push(sample);
+                    if (sample.timestamp > nextTimestampWeWant) {
+                        expectedFrames.push(timestampTargets.shift());
+                        while (queued.length > 0) {
+                            const sam = queued.shift();
+                            await decoder.waitForQueueToBeLessThan(10);
+                            await decoder.decode(sam);
+                        }
+                        if (timestampTargets.length === 0) {
+                            await decoder.flush();
+                            controller.abort();
+                        }
+                    }
+                };
+            },
+        });
+    }
+    catch (e) {
+        if (!(0, media_parser_1.hasBeenAborted)(e)) {
+            throw e;
+        }
+    }
+};
+exports.extractFrames = extractFrames;

package/dist/esm/index.mjs

@@ -4185,29 +4185,26 @@ var getBytesPerSample = (sampleFormat) => {
   if (sampleFormat === "s16") {
     return 2;
   }
-  if (sampleFormat === "s32") {
+  if (sampleFormat === "s24") {
     return 4;
   }
-  if (sampleFormat === "f32") {
-    return 4;
-  }
-  if (sampleFormat === "u8") {
-    return 1;
-  }
-  if (sampleFormat === "f32-planar") {
-    return 4;
-  }
-  if (sampleFormat === "s16-planar") {
-    return 2;
-  }
-  if (sampleFormat === "s32-planar") {
-    return 4;
-  }
-  if (sampleFormat === "u8-planar") {
-    return 1;
-  }
   throw new Error(`Unsupported sample format: ${sampleFormat}`);
 };
+function uint8_24le_to_uint32(u8) {
+  if (u8.length % 3 !== 0) {
+    throw new Error("Input length must be a multiple of 3");
+  }
+  const count = u8.length / 3;
+  const out = new Uint32Array(count);
+  let j = 0;
+  for (let i = 0;i < count; i++) {
+    const b0 = u8[j++];
+    const b1 = u8[j++];
+    const b2 = u8[j++];
+    out[i] = (b0 | b1 << 8 | b2 << 16) << 8;
+  }
+  return out;
+}
 var getAudioData = (audioSample) => {
   if (audioSample instanceof EncodedAudioChunk) {
     const data = new Uint8Array(audioSample.byteLength);
@@ -4225,12 +4222,14 @@ var getWaveAudioDecoder = ({
 }) => {
   const processSample = async (audioSample) => {
     const bytesPerSample = getBytesPerSample(sampleFormat);
-    const data = getAudioData(audioSample);
+    const rawData = getAudioData(audioSample);
+    const data = sampleFormat === "s24" && rawData instanceof Uint8Array ? uint8_24le_to_uint32(rawData) : rawData;
+    const numberOfFrames = data.byteLength / bytesPerSample / config.numberOfChannels;
     const audioData = new AudioData({
       data,
-      format: sampleFormat,
+      format: sampleFormat === "s16" ? "s16" : "s32",
       numberOfChannels: config.numberOfChannels,
-      numberOfFrames: data.byteLength / bytesPerSample / config.numberOfChannels,
+      numberOfFrames,
       sampleRate: config.sampleRate,
       timestamp: audioSample.timestamp
     });
@@ -4321,6 +4320,16 @@ var internalCreateAudioDecoder = async ({
       onError
     });
   }
+  if (config.codec === "pcm-s24") {
+    return getWaveAudioDecoder({
+      onFrame,
+      config,
+      sampleFormat: "s24",
+      logLevel,
+      ioSynchronizer,
+      onError
+    });
+  }
   const audioDecoder = new AudioDecoder({
     async output(frame) {
       try {

package/dist/get-wave-audio-decoder.d.ts

@@ -2,7 +2,7 @@ import type { MediaParserLogLevel } from '@remotion/media-parser';
 import type { CreateAudioDecoderInit, WebCodecsAudioDecoder } from './create-audio-decoder';
 import type { IoSynchronizer } from './io-manager/io-synchronizer';
 export declare const getWaveAudioDecoder: ({ onFrame, config, sampleFormat, ioSynchronizer, onError, }: Pick<CreateAudioDecoderInit, "onFrame" | "config"> & {
-    sampleFormat: AudioSampleFormat;
+    sampleFormat: "s16" | "s24";
     logLevel: MediaParserLogLevel;
     ioSynchronizer: IoSynchronizer;
     onError: (error: Error) => void;

package/dist/get-wave-audio-decoder.js

@@ -5,29 +5,26 @@ const getBytesPerSample = (sampleFormat) => {
     if (sampleFormat === 's16') {
         return 2;
     }
-    if (sampleFormat === 's32') {
+    if (sampleFormat === 's24') {
         return 4;
     }
-    if (sampleFormat === 'f32') {
-        return 4;
-    }
-    if (sampleFormat === 'u8') {
-        return 1;
-    }
-    if (sampleFormat === 'f32-planar') {
-        return 4;
-    }
-    if (sampleFormat === 's16-planar') {
-        return 2;
-    }
-    if (sampleFormat === 's32-planar') {
-        return 4;
-    }
-    if (sampleFormat === 'u8-planar') {
-        return 1;
-    }
     throw new Error(`Unsupported sample format: ${sampleFormat}`);
 };
+function uint8_24le_to_uint32(u8) {
+    if (u8.length % 3 !== 0) {
+        throw new Error('Input length must be a multiple of 3');
+    }
+    const count = u8.length / 3;
+    const out = new Uint32Array(count);
+    let j = 0;
+    for (let i = 0; i < count; i++) {
+        const b0 = u8[j++];
+        const b1 = u8[j++];
+        const b2 = u8[j++];
+        out[i] = (b0 | (b1 << 8) | (b2 << 16)) << 8;
+    }
+    return out;
+}
 const getAudioData = (audioSample) => {
     if (audioSample instanceof EncodedAudioChunk) {
         const data = new Uint8Array(audioSample.byteLength);
@@ -39,12 +36,16 @@ const getAudioData = (audioSample) => {
 const getWaveAudioDecoder = ({ onFrame, config, sampleFormat, ioSynchronizer, onError, }) => {
     const processSample = async (audioSample) => {
         const bytesPerSample = getBytesPerSample(sampleFormat);
-        const data = getAudioData(audioSample);
+        const rawData = getAudioData(audioSample);
+        const data = sampleFormat === 's24' && rawData instanceof Uint8Array
+            ? uint8_24le_to_uint32(rawData)
+            : rawData;
+        const numberOfFrames = data.byteLength / bytesPerSample / config.numberOfChannels;
         const audioData = new AudioData({
             data,
-            format: sampleFormat,
+            format: sampleFormat === 's16' ? 's16' : 's32',
             numberOfChannels: config.numberOfChannels,
-            numberOfFrames: data.byteLength / bytesPerSample / config.numberOfChannels,
+            numberOfFrames,
             sampleRate: config.sampleRate,
             timestamp: audioSample.timestamp,
         });

package/dist/video-decoder.d.ts

@@ -0,0 +1,23 @@
+import type { MediaParserLogLevel, MediaParserVideoSample } from '@remotion/media-parser';
+import type { WebCodecsController } from './webcodecs-controller';
+export type WebCodecsVideoDecoder = {
+    decode: (videoSample: MediaParserVideoSample | EncodedVideoChunk) => void;
+    close: () => void;
+    flush: () => Promise<void>;
+    waitForFinish: () => Promise<void>;
+    waitForQueueToBeLessThan: (items: number) => Promise<void>;
+};
+export declare const internalCreateVideoDecoder: ({ onFrame, onError, controller, config, logLevel, }: {
+    onFrame: (frame: VideoFrame) => Promise<void> | void;
+    onError: (error: Error) => void;
+    controller: WebCodecsController | null;
+    config: VideoDecoderConfig;
+    logLevel: MediaParserLogLevel;
+}) => WebCodecsVideoDecoder;
+export declare const createVideoDecoder: ({ onFrame, onError, controller, track, logLevel, }: {
+    track: VideoDecoderConfig;
+    onFrame: (frame: VideoFrame) => Promise<void> | void;
+    onError: (error: Error) => void;
+    controller?: WebCodecsController;
+    logLevel?: MediaParserLogLevel;
+}) => WebCodecsVideoDecoder;

package/dist/video-decoder.js

@@ -0,0 +1,80 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createVideoDecoder = exports.internalCreateVideoDecoder = void 0;
+const io_synchronizer_1 = require("./io-manager/io-synchronizer");
+const log_1 = require("./log");
+const internalCreateVideoDecoder = ({ onFrame, onError, controller, config, logLevel, }) => {
+    const ioSynchronizer = (0, io_synchronizer_1.makeIoSynchronizer)({
+        logLevel,
+        label: 'Video decoder',
+        controller,
+    });
+    const videoDecoder = new VideoDecoder({
+        async output(frame) {
+            try {
+                await onFrame(frame);
+            }
+            catch (err) {
+                onError(err);
+                frame.close();
+            }
+            ioSynchronizer.onOutput(frame.timestamp);
+        },
+        error(error) {
+            onError(error);
+        },
+    });
+    const close = () => {
+        if (controller) {
+            controller._internals._mediaParserController._internals.signal.removeEventListener('abort', 
+            // eslint-disable-next-line @typescript-eslint/no-use-before-define
+            onAbort);
+        }
+        if (videoDecoder.state === 'closed') {
+            return;
+        }
+        videoDecoder.close();
+    };
+    const onAbort = () => {
+        close();
+    };
+    if (controller) {
+        controller._internals._mediaParserController._internals.signal.addEventListener('abort', onAbort);
+    }
+    videoDecoder.configure(config);
+    const decode = (sample) => {
+        if (videoDecoder.state === 'closed') {
+            return;
+        }
+        const encodedChunk = sample instanceof EncodedVideoChunk
+            ? sample
+            : new EncodedVideoChunk(sample);
+        videoDecoder.decode(encodedChunk);
+        ioSynchronizer.inputItem(sample.timestamp);
+    };
+    return {
+        decode,
+        waitForFinish: async () => {
+            await videoDecoder.flush();
+            log_1.Log.verbose(logLevel, 'Flushed video decoder');
+            await ioSynchronizer.waitForFinish();
+            log_1.Log.verbose(logLevel, 'IO synchro finished');
+        },
+        close,
+        flush: async () => {
+            await videoDecoder.flush();
+        },
+        waitForQueueToBeLessThan: ioSynchronizer.waitForQueueSize,
+    };
+};
+exports.internalCreateVideoDecoder = internalCreateVideoDecoder;
+const createVideoDecoder = ({ onFrame, onError, controller, track, logLevel, }) => {
+    return (0, exports.internalCreateVideoDecoder)({
+        onFrame,
+        onError,
+        controller: controller ?? null,
+        config: track,
+        logLevel: logLevel ?? 'info',
+    });
+};
+exports.createVideoDecoder = createVideoDecoder;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remotion/webcodecs",
-  "version": "4.0.344",
+  "version": "4.0.345",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "module": "dist/esm/index.mjs",
@@ -19,18 +19,18 @@
   "author": "Jonny Burger <jonny@remotion.dev>",
   "license": "Remotion License (See https://remotion.dev/docs/webcodecs#license)",
   "dependencies": {
-    "@remotion/media-parser": "4.0.344",
-    "@remotion/licensing": "4.0.344"
+    "@remotion/media-parser": "4.0.345",
+    "@remotion/licensing": "4.0.345"
   },
   "peerDependencies": {},
   "devDependencies": {
     "@types/dom-webcodecs": "0.1.11",
     "playwright": "1.51.1",
-    "vite": "5.4.19",
+    "vite": "5.4.20",
     "@playwright/test": "1.51.1",
     "eslint": "9.19.0",
-    "@remotion/example-videos": "4.0.344",
-    "@remotion/eslint-config-internal": "4.0.344"
+    "@remotion/eslint-config-internal": "4.0.345",
+    "@remotion/example-videos": "4.0.345"
   },
   "keywords": [],
   "publishConfig": {