@mediabunny/server 1.45.4 → 1.46.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/server",
   "author": "Vanilagy",
-  "version": "1.45.4",
+  "version": "1.46.0",
   "description": "Adds full video and audio decoder and encoder support to Mediabunny for use in server-side environments (Node, Bun, Deno). Based on NodeAV.",
   "main": "./dist/bundles/mediabunny-server.cjs",
   "module": "./dist/bundles/mediabunny-server.mjs",

package/src/audio-decoder.ts

@@ -15,7 +15,7 @@ import { AvFrameAudioSampleResource } from './audio-sample';
 export class NodeAvAudioDecoder extends CustomAudioDecoder {
 	frame!: NodeAv.Frame;
 	packet!: NodeAv.Packet;
-	codecContext!: NodeAv.CodecContext;
+	codecContext: NodeAv.CodecContext | null = null;
 
 	// eslint-disable-next-line @typescript-eslint/no-unused-vars
 	static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean {
@@ -61,6 +61,8 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
 	}
 
 	async decode(packet: EncodedPacket): Promise<void> {
+		assert(this.codecContext);
+
 		this.packet.isKeyframe = packet.type === 'key';
 		this.packet.data = Buffer.from(packet.data);
 		this.packet.timeBase = { num: 1, den: this.config.sampleRate };
@@ -71,6 +73,8 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
 		const ret = await this.codecContext.sendPacket(this.packet);
 		NodeAv.FFmpegError.throwIfError(ret, 'Send packet');
 
+		this.packet.unref(); // Don't need the data again, so just unref it
+
 		while (true) {
 			const receiveRet = await this.codecContext.receiveFrame(this.frame);
 			if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
@@ -94,6 +98,8 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
 	}
 
 	async flush(): Promise<void> {
+		assert(this.codecContext);
+
 		// Send null packet to signal flush
 		const ret = await this.codecContext.sendPacket(null);
 		NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder');
@@ -113,7 +119,7 @@ export class NodeAvAudioDecoder extends CustomAudioDecoder {
 	}
 
 	close(): MaybePromise<void> {
-		this.codecContext.freeContext();
+		this.codecContext?.freeContext();
 		this.frame.free();
 		this.packet.free();
 	}

package/src/audio-encoder.ts

@@ -35,16 +35,16 @@ const FRAME_SIZE_FALLBACK = 1024; // Just 'cause
 export class NodeAvAudioEncoder extends CustomAudioEncoder {
 	frame!: NodeAv.Frame;
 	packet!: NodeAv.Packet;
-	avCodec!: NodeAv.Codec;
 	codecContext: NodeAv.CodecContext | null = null;
+	resampler: NodeAv.SoftwareResampleContext | null = null;
+	dstFrame: NodeAv.Frame | null = null;
+	avCodec!: NodeAv.Codec;
 	firstExpectedTimestamp: number | null = null;
 	outputTimestampOffset = 0;
 
-	resampler: NodeAv.SoftwareResampleContext | null = null;
 	inputParametersKey: string | null = null;
 	resamplerInputSampleRate: number | null = null;
 	nextResamplerPts: bigint | null = null;
-	dstFrame: NodeAv.Frame | null = null;
 	packetEmitted = false;
 	adtsHeaderTemplate: AdtsHeaderTemplate | null = null;
 
@@ -134,6 +134,10 @@ export class NodeAvAudioEncoder extends CustomAudioEncoder {
 		this.firstExpectedTimestamp ??= audioSample.timestamp;
 
 		if (audioSample._data instanceof AvFrameAudioSampleResource) {
+			// Release any buffers still referenced from the previous encode before reffing the new frame, otherwise
+			// av_frame_ref leaks them
+			// https://github.com/Vanilagy/mediabunny/issues/392
+			this.frame.unref();
 			this.frame.ref(audioSample._data.frame);
 		} else {
 			copyAudioSampleToAvFrame(audioSample, this.frame);

package/src/audio-sample.ts

@@ -42,6 +42,9 @@ export class AvFrameAudioSampleResource extends AudioSampleResource {
 	constructor(frame: NodeAv.Frame) {
 		super();
 
+		if (!(frame instanceof NodeAv.Frame)) {
+			throw new TypeError('frame must be a NodeAv.Frame.');
+		}
 		if (frame.getMediaType() !== NodeAv.AVMEDIA_TYPE_AUDIO) {
 			throw new Error('AvFrameAudioSampleResource must be initialized with an audio frame.');
 		}

package/src/index.ts

@@ -6,7 +6,14 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-import { AudioSample, registerDecoder, registerEncoder, registerVideoSampleTransformer, VideoSample } from 'mediabunny';
+import {
+	AudioSample,
+	MaybePromise,
+	registerDecoder,
+	registerEncoder,
+	registerVideoSampleTransformer,
+	VideoSample,
+} from 'mediabunny';
 import * as NodeAv from 'node-av';
 import { NodeAvVideoDecoder } from './video-decoder';
 import { NodeAvVideoEncoder } from './video-encoder';
@@ -28,6 +35,29 @@ if ((globalThis as Record<symbol, unknown>)[SERVER_LOADED_SYMBOL]) {
 
 let registered = false;
 
+/**
+ * Options for configuring Mediabunny's server-side polyfills.
+ * @group \@mediabunny/server
+ * @public
+ */
+type MediabunnyServerOptions = {
+	/**
+	 * The hardware context to use for hardware-accelerated decoding and encoding. When set to a
+	 * `NodeAv.HardwareContext`, that context is used directly. When set to a function, the function is invoked (with
+	 * no caching) every time a hardware decoder or encoder codec is resolved, receiving the codec ID and returning the
+	 * context to use for it (or `null` for no hardware acceleration).
+	 *
+	 * When not set, a context is automatically detected on first use and then cached.
+	 */
+	hardwareContext?:
+		| NodeAv.HardwareContext
+		| null
+		| ((codecId: NodeAv.AVCodecID) => MaybePromise<NodeAv.HardwareContext | null>);
+};
+
+/** @internal */
+export let _serverOptions: MediabunnyServerOptions = {};
+
 /**
  * Registers video and audio decoders and encoders for all codecs, using FFmpeg's libavcodec under the hood.
  * Additionally, a custom `VideoSample` transformer based on libavfilter is registered to enable resizing, rotation and
@@ -38,14 +68,32 @@ let registered = false;
  * The decoders and encoders will automatically detect hardware acceleration support for each codec and platform and
  * make use of it if applicable.
  *
+ * You can pass optional {@link MediabunnyServerOptions} for additional configuration.
+ *
  * @group \@mediabunny/server
  * @public
  */
-export const registerMediabunnyServer = () => {
+export const registerMediabunnyServer = (options: MediabunnyServerOptions = {}) => {
+	if (typeof options !== 'object' || !options) {
+		throw new TypeError('options must be an object.');
+	}
+	if (
+		options.hardwareContext != null
+		&& !(
+			options.hardwareContext instanceof NodeAv.HardwareContext || typeof options.hardwareContext === 'function'
+		)
+	) {
+		throw new TypeError(
+			'options.hardwareContext, when provided, must be a NodeAv.HardwareContext, a function, or null.',
+		);
+	}
+
 	if (registered) {
 		return;
 	}
+
 	registered = true;
+	_serverOptions = options;
 
 	NodeAv.Log.setLevel(NodeAv.AV_LOG_ERROR);
 
@@ -60,6 +108,8 @@ export const registerMediabunnyServer = () => {
 	registerVideoSampleTransformer(transformVideoSample);
 };
 
+export type { MediabunnyServerOptions };
+
 export { AvFrameVideoSampleResource } from './video-sample';
 export { AvFrameAudioSampleResource } from './audio-sample';
 
@@ -76,8 +126,17 @@ export { AvFrameAudioSampleResource } from './audio-sample';
  * @public
  */
 export const toAvFrame = async (sample: VideoSample | AudioSample, frame: NodeAv.Frame) => {
+	if (!(sample instanceof VideoSample) && !(sample instanceof AudioSample)) {
+		throw new TypeError('sample must be a VideoSample or an AudioSample.');
+	}
+	if (!(frame instanceof NodeAv.Frame)) {
+		throw new TypeError('frame must be a NodeAv.Frame.');
+	}
+
 	if (sample instanceof VideoSample) {
 		if (sample._data instanceof AvFrameVideoSampleResource) {
+			// We're overriding the frame, so release whatever it referenced before, otherwise av_frame_ref leaks it
+			frame.unref();
 			frame.ref(sample._data.frame);
 		} else {
 			if (sample.format === null) {
@@ -92,6 +151,8 @@ export const toAvFrame = async (sample: VideoSample | AudioSample, frame: NodeAv
 		frame.timeBase = new NodeAv.Rational(1, 1e6);
 	} else {
 		if (sample._data instanceof AvFrameAudioSampleResource) {
+			// We're overriding the frame, so release whatever it referenced before, otherwise av_frame_ref leaks it
+			frame.unref();
 			frame.ref(sample._data.frame);
 		} else {
 			copyAudioSampleToAvFrame(sample, frame);

package/src/misc.ts

@@ -8,6 +8,7 @@
 
 import { VideoSamplePixelFormat, MediaCodec } from 'mediabunny';
 import * as NodeAv from 'node-av';
+import { _serverOptions } from './index';
 
 export const CODEC_TO_CODEC_ID: Partial<Record<MediaCodec, NodeAv.AVCodecID>> = {
 	avc: NodeAv.AV_CODEC_ID_H264,
@@ -27,14 +28,35 @@ export const CODEC_TO_CODEC_ID: Partial<Record<MediaCodec, NodeAv.AVCodecID>> =
 
 let cachedHardwareContext: NodeAv.HardwareContext | null | undefined = undefined;
 export const getHardwareContext = (): NodeAv.HardwareContext | null => {
+	if (_serverOptions.hardwareContext !== undefined && typeof _serverOptions.hardwareContext !== 'function') {
+		// Return the user-provided one
+		return _serverOptions.hardwareContext;
+	}
+
 	if (cachedHardwareContext === undefined) {
 		cachedHardwareContext = NodeAv.HardwareContext.auto();
 	}
 	return cachedHardwareContext;
 };
 
+const validateHwContext = (hw: NodeAv.HardwareContext | null) => {
+	if (hw !== null && !(hw instanceof NodeAv.HardwareContext)) {
+		throw new TypeError(
+			'When serverOptions.hardwareContext is a function, it must return or resolve to a'
+			+ ' NodeAv.HardwareContext or null.',
+		);
+	}
+};
+
 const hardwareDecoderCodecCache = new Map<NodeAv.AVCodecID, NodeAv.Codec | null>();
-export const getHardwareDecoderCodec = (codecId: NodeAv.AVCodecID): NodeAv.Codec | null => {
+export const getHardwareDecoderCodec = async (codecId: NodeAv.AVCodecID): Promise<NodeAv.Codec | null> => {
+	if (typeof _serverOptions.hardwareContext === 'function') {
+		const hw = await _serverOptions.hardwareContext(codecId);
+		validateHwContext(hw);
+
+		return hw?.getDecoderCodec(codecId) ?? null;
+	}
+
 	if (!hardwareDecoderCodecCache.has(codecId)) {
 		const hw = getHardwareContext();
 		hardwareDecoderCodecCache.set(codecId, hw?.getDecoderCodec(codecId) ?? null);
@@ -43,7 +65,14 @@ export const getHardwareDecoderCodec = (codecId: NodeAv.AVCodecID): NodeAv.Codec
 };
 
 const hardwareEncoderCodecCache = new Map<NodeAv.AVCodecID, NodeAv.Codec | null>();
-export const getHardwareEncoderCodec = (codecId: NodeAv.AVCodecID): NodeAv.Codec | null => {
+export const getHardwareEncoderCodec = async (codecId: NodeAv.AVCodecID): Promise<NodeAv.Codec | null> => {
+	if (typeof _serverOptions.hardwareContext === 'function') {
+		const hw = await _serverOptions.hardwareContext(codecId);
+		validateHwContext(hw);
+
+		return hw?.getEncoderCodec(codecId) ?? null;
+	}
+
 	if (!hardwareEncoderCodecCache.has(codecId)) {
 		const hw = getHardwareContext();
 		hardwareEncoderCodecCache.set(codecId, hw?.getEncoderCodec(codecId) ?? null);

package/src/video-decoder.ts

@@ -58,7 +58,7 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
 		) {
 			codec = NodeAv.Codec.findDecoder(codecId);
 		} else {
-			codec = getHardwareDecoderCodec(codecId) ?? NodeAv.Codec.findDecoder(codecId);
+			codec = (await getHardwareDecoderCodec(codecId)) ?? NodeAv.Codec.findDecoder(codecId);
 		}
 
 		if (!codec) {
@@ -91,9 +91,10 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
 	async decode(packet: EncodedPacket) {
 		if (this.codecContext === null) {
 			await this.initCodecContext(packet);
-			assert(this.codecContext);
 		}
 
+		assert(this.codecContext);
+
 		this.packet.isKeyframe = packet.type === 'key';
 		this.packet.data = Buffer.from(packet.data);
 		this.packet.timeBase = { num: 1, den: 1e6 };
@@ -144,6 +145,8 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
 		const ret = await this.codecContext.sendPacket(this.packet);
 		NodeAv.FFmpegError.throwIfError(ret, 'Send packet');
 
+		this.packet.unref(); // Don't need the data again, so just unref it
+
 		while (true) {
 			const receiveRet = await this.codecContext.receiveFrame(this.frame);
 			if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {

package/src/video-encoder.ts

@@ -44,14 +44,13 @@ import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from
 export class NodeAvVideoEncoder extends CustomVideoEncoder {
 	frame!: NodeAv.Frame;
 	packet!: NodeAv.Packet;
-	avCodec!: NodeAv.Codec;
 	codecContext: NodeAv.CodecContext | null = null;
+	scaler: NodeAv.SoftwareScaleContext | null = null;
+	dstFrame: NodeAv.Frame | null = null;
+	avCodec!: NodeAv.Codec;
 	lastBuffer: Buffer | null = null;
 	packetEmitted = false;
-
-	scaler: NodeAv.SoftwareScaleContext | null = null;
 	lastScalerKey: string | null = null;
-	dstFrame: NodeAv.Frame | null = null;
 
 	// Bookkeeping to restore the original timing information
 	preciseTimings: {
@@ -84,7 +83,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 		} else if (this.config.hardwareAcceleration === 'prefer-software') {
 			codec = NodeAv.Codec.findEncoder(codecId);
 		} else {
-			codec = getHardwareEncoderCodec(codecId) ?? NodeAv.Codec.findEncoder(codecId);
+			codec = (await getHardwareEncoderCodec(codecId)) ?? NodeAv.Codec.findEncoder(codecId);
 		}
 
 		if (!codec) {
@@ -183,10 +182,14 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 	async encode(videoSample: VideoSample, options: VideoEncoderEncodeOptions): Promise<void> {
 		if (this.codecContext === null) {
 			await this.createCodecContext();
-			assert(this.codecContext);
 		}
+		assert(this.codecContext);
 
 		if (videoSample._data instanceof AvFrameVideoSampleResource) {
+			// Release any buffers still referenced from the previous encode before reffing the new frame, otherwise
+			// av_frame_ref leaks them
+			// https://github.com/Vanilagy/mediabunny/issues/392
+			this.frame.unref();
 			this.frame.ref(videoSample._data.frame);
 		} else {
 			if (videoSample.format === null) {
@@ -513,12 +516,10 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder {
 				displayAspectHeight: this.config.displayHeight ?? this.codecContext.height,
 				description: decoderConfigDescription ?? undefined,
 				colorSpace: {
-					primaries:
-							unmapColorPrimaries(this.codecContext.colorPrimaries) as VideoColorPrimaries,
-					matrix:
-							unmapMatrixCoefficients(this.codecContext.colorSpace) as VideoMatrixCoefficients,
+					primaries: unmapColorPrimaries(this.codecContext.colorPrimaries) as VideoColorPrimaries,
+					matrix: unmapMatrixCoefficients(this.codecContext.colorSpace) as VideoMatrixCoefficients,
 					transfer:
-							unmapTransferCharacteristics(this.codecContext.colorTrc) as VideoTransferCharacteristics,
+						unmapTransferCharacteristics(this.codecContext.colorTrc) as VideoTransferCharacteristics,
 					fullRange: this.codecContext.colorRange === NodeAv.AVCOL_RANGE_JPEG
 						? true
 						: this.codecContext.colorRange === NodeAv.AVCOL_RANGE_MPEG

package/src/video-sample.ts

@@ -73,6 +73,9 @@ export class AvFrameVideoSampleResource extends VideoSampleResource {
 	constructor(frame: NodeAv.Frame) {
 		super();
 
+		if (!(frame instanceof NodeAv.Frame)) {
+			throw new TypeError('frame must be a NodeAv.Frame.');
+		}
 		if (frame.getMediaType() !== NodeAv.AVMEDIA_TYPE_VIDEO) {
 			throw new Error('AvFrameVideoSampleResource must be initialized with a video frame.');
 		}