@mediabunny/server 1.45.0

package/src/audio-decoder.ts

@@ -0,0 +1,120 @@
+/*!
+ * Copyright (c) 2026-present, Vanilagy and contributors
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+import { AudioCodec, AudioSample, CustomAudioDecoder, EncodedPacket, type MaybePromise } from 'mediabunny';
+import * as NodeAv from 'node-av';
+import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc';
+import { assert, toUint8Array } from '../../../src/misc';
+import { AvFrameAudioSampleResource } from './audio-sample';
+
+export class NodeAvAudioDecoder extends CustomAudioDecoder {
+	frame!: NodeAv.Frame;
+	packet!: NodeAv.Packet;
+	codecContext!: NodeAv.CodecContext;
+
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean {
+		return codec === 'aac'
+			|| codec === 'opus'
+			|| codec === 'mp3'
+			|| codec === 'vorbis'
+			|| codec === 'flac'
+			|| codec === 'ac3'
+			|| codec === 'eac3';
+	}
+
+	async init(): Promise<void> {
+		this.frame = new NodeAv.Frame();
+		this.frame.alloc();
+		this.packet = new NodeAv.Packet();
+		this.packet.alloc();
+
+		const codecId = CODEC_TO_CODEC_ID[this.codec];
+		assert(codecId !== undefined);
+
+		const codec = NodeAv.Codec.findDecoder(codecId);
+		if (codec === null) {
+			throw new Error(`Unable to obtain libav codec for '${this.codec}'.`);
+		}
+
+		const codecContext = new NodeAv.CodecContext();
+		codecContext.allocContext3(codec);
+
+		codecContext.sampleRate = this.config.sampleRate;
+		codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels);
+		codecContext.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
+		codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO;
+		codecContext.codecId = codecId;
+		codecContext.extraData = this.config.description
+			? Buffer.from(toUint8Array(this.config.description))
+			: null;
+
+		const ret = await codecContext.open2();
+		NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
+
+		this.codecContext = codecContext;
+	}
+
+	async decode(packet: EncodedPacket): Promise<void> {
+		this.packet.isKeyframe = packet.type === 'key';
+		this.packet.data = Buffer.from(packet.data);
+		this.packet.timeBase = { num: 1, den: this.config.sampleRate };
+		this.packet.pts = BigInt(Math.round(packet.timestamp * this.config.sampleRate));
+		this.packet.dts = NodeAv.AV_NOPTS_VALUE;
+		this.packet.duration = BigInt(Math.round(packet.duration * this.config.sampleRate));
+
+		const ret = await this.codecContext.sendPacket(this.packet);
+		NodeAv.FFmpegError.throwIfError(ret, 'Send packet');
+
+		while (true) {
+			const receiveRet = await this.codecContext.receiveFrame(this.frame);
+			if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
+				break;
+			}
+
+			this.receiveFrame(receiveRet);
+		}
+	}
+
+	receiveFrame(ret: number) {
+		NodeAv.FFmpegError.throwIfError(ret, 'Receive frame');
+
+		const clone = this.frame.clone();
+		if (!clone) {
+			throw new Error('Allocation failure during frame clone.');
+		}
+
+		clone.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
+		this.onSample(new AudioSample(new AvFrameAudioSampleResource(clone)));
+	}
+
+	async flush(): Promise<void> {
+		// Send null packet to signal flush
+		const ret = await this.codecContext.sendPacket(null);
+		NodeAv.FFmpegError.throwIfError(ret, 'Flush decoder');
+
+		// Keep receiving frames until no more are available
+		while (true) {
+			const receiveRet = await this.codecContext.receiveFrame(this.frame);
+			if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
+				// No more frames available
+				break;
+			}
+
+			this.receiveFrame(receiveRet);
+		}
+
+		this.codecContext.flushBuffers();
+	}
+
+	close(): MaybePromise<void> {
+		this.codecContext.freeContext();
+		this.frame.free();
+		this.packet.free();
+	}
+}

package/src/audio-encoder.ts

@@ -0,0 +1,396 @@
+/*!
+ * Copyright (c) 2026-present, Vanilagy and contributors
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+import {
+	AudioCodec,
+	AudioSample,
+	CustomAudioEncoder,
+	type MaybePromise,
+	QUALITY_MEDIUM,
+	EncodedPacket,
+} from 'mediabunny';
+import * as NodeAv from 'node-av';
+import { CODEC_TO_CODEC_ID, getChannelLayout } from './misc';
+import { assert, toUint8Array } from '../../../src/misc';
+import { copyAudioSampleToAvFrame, AvFrameAudioSampleResource } from './audio-sample';
+import {
+	AdtsHeaderTemplate,
+	buildAdtsHeaderTemplate,
+	parseAacAudioSpecificConfig,
+} from '../../../shared/aac-misc';
+
+const AAC_SAMPLE_RATES
+	= [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350];
+const OPUS_SAMPLE_RATES = [8000, 12000, 16000, 24000, 48000];
+const MP3_SAMPLE_RATES = [8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000];
+const AC3_SAMPLE_RATES = [32000, 44100, 48000];
+
+const FRAME_SIZE_FALLBACK = 1024; // Just 'cause
+
+export class NodeAvAudioEncoder extends CustomAudioEncoder {
+	frame!: NodeAv.Frame;
+	packet!: NodeAv.Packet;
+	avCodec!: NodeAv.Codec;
+	codecContext: NodeAv.CodecContext | null = null;
+	firstExpectedTimestamp: number | null = null;
+	outputTimestampOffset = 0;
+
+	resampler: NodeAv.SoftwareResampleContext | null = null;
+	inputParametersKey: string | null = null;
+	resamplerInputSampleRate: number | null = null;
+	nextResamplerPts: bigint | null = null;
+	dstFrame: NodeAv.Frame | null = null;
+	packetEmitted = false;
+	adtsHeaderTemplate: AdtsHeaderTemplate | null = null;
+
+	static override supports(codec: AudioCodec, config: AudioEncoderConfig): boolean {
+		const { numberOfChannels, sampleRate } = config;
+
+		return (
+			codec === 'aac' && numberOfChannels >= 1 && numberOfChannels <= 48
+			&& AAC_SAMPLE_RATES.includes(sampleRate)
+		) || (
+			codec === 'opus' && numberOfChannels >= 1 && numberOfChannels <= 255
+			&& OPUS_SAMPLE_RATES.includes(sampleRate)
+		) || (
+			codec === 'mp3' && numberOfChannels >= 1 && numberOfChannels <= 2
+			&& MP3_SAMPLE_RATES.includes(sampleRate)
+		) || (
+			codec === 'vorbis' && numberOfChannels >= 1 && numberOfChannels <= 255
+			&& sampleRate <= 200000
+		) || (
+			codec === 'flac' && numberOfChannels >= 1 && numberOfChannels <= 8
+			&& sampleRate <= 655350
+		) || (
+			codec === 'ac3' && numberOfChannels >= 1 && numberOfChannels <= 6
+			&& AC3_SAMPLE_RATES.includes(sampleRate)
+		) || (
+			codec === 'eac3' && numberOfChannels >= 1 && numberOfChannels <= 16
+			&& AC3_SAMPLE_RATES.includes(sampleRate)
+		);
+	}
+
+	async init(): Promise<void> {
+		this.frame = new NodeAv.Frame();
+		this.frame.alloc();
+		this.packet = new NodeAv.Packet();
+		this.packet.alloc();
+
+		const codecId = CODEC_TO_CODEC_ID[this.codec];
+		assert(codecId !== undefined);
+
+		const codec = NodeAv.Codec.findEncoder(codecId);
+		if (!codec) {
+			throw new Error(`Unable to obtain libav codec for '${this.codec}'.`);
+		}
+
+		this.avCodec = codec;
+
+		await this.createCodecContext();
+	}
+
+	async createCodecContext() {
+		assert(this.codecContext === null);
+
+		const codecContext = new NodeAv.CodecContext();
+		codecContext.allocContext3(this.avCodec);
+
+		let sampleFormat = NodeAv.AV_SAMPLE_FMT_FLTP;
+		if (this.avCodec.sampleFormats && !this.avCodec.sampleFormats.includes(NodeAv.AV_SAMPLE_FMT_FLTP)) {
+			// Use a format that's supported
+			sampleFormat = this.avCodec.sampleFormats[0]!;
+		}
+
+		codecContext.sampleRate = this.config.sampleRate;
+		codecContext.channelLayout = getChannelLayout(this.config.numberOfChannels);
+		codecContext.codecType = NodeAv.AVMEDIA_TYPE_AUDIO;
+		codecContext.codecId = CODEC_TO_CODEC_ID[this.codec]!;
+		codecContext.sampleFormat = sampleFormat;
+		codecContext.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
+		codecContext.bitRate = BigInt(this.config.bitrate ?? QUALITY_MEDIUM._toAudioBitrate(this.codec) ?? 0);
+
+		if (this.config.bitrateMode === 'constant') {
+			codecContext.rcMinRate = codecContext.bitRate;
+			codecContext.rcMaxRate = codecContext.bitRate;
+		}
+
+		const ret = await codecContext.open2();
+		NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
+
+		this.codecContext = codecContext;
+	}
+
+	async encode(audioSample: AudioSample): Promise<void> {
+		if (this.codecContext === null) {
+			await this.createCodecContext();
+			assert(this.codecContext);
+		}
+
+		this.firstExpectedTimestamp ??= audioSample.timestamp;
+
+		if (audioSample._data instanceof AvFrameAudioSampleResource) {
+			this.frame.ref(audioSample._data.frame);
+		} else {
+			copyAudioSampleToAvFrame(audioSample, this.frame);
+		}
+
+		this.frame.pts = BigInt(Math.round(audioSample.timestamp * this.config.sampleRate));
+		this.frame.duration = BigInt(Math.round(audioSample.duration * this.config.sampleRate));
+		this.frame.timeBase = new NodeAv.Rational(1, this.config.sampleRate);
+
+		const key = `${this.frame.sampleRate}:${this.frame.channels}:${this.frame.format}`;
+		if (this.inputParametersKey !== null && this.inputParametersKey !== key) {
+			throw new Error(
+				'Input audio parameters changed. For this audio encoder, you cannot change the input audio'
+				+ ' parameters over time.',
+			);
+		}
+		this.inputParametersKey = key;
+
+		// We need the resampler when:
+		// 1. Format conversion is needed (sample format, sample rate, or channel count differs)
+		// 2. The codec requires fixed frame sizes
+		const requiresResampler
+			= this.codecContext.frameSize > 0
+				|| this.codecContext.sampleFormat !== this.frame.format
+				|| this.codecContext.sampleRate !== this.frame.sampleRate
+				|| this.codecContext.channels !== this.frame.channels;
+
+		if (requiresResampler) {
+			if (!this.resampler) {
+				this.resampler = new NodeAv.SoftwareResampleContext();
+				this.resamplerInputSampleRate = this.frame.sampleRate;
+
+				const outLayout = getChannelLayout(this.codecContext.channels);
+				const inLayout = getChannelLayout(this.frame.channels);
+
+				const ret = this.resampler.allocSetOpts2(
+					outLayout, this.codecContext.sampleFormat, this.codecContext.sampleRate,
+					inLayout, this.frame.format as NodeAv.AVSampleFormat, this.frame.sampleRate,
+				);
+				NodeAv.FFmpegError.throwIfError(ret, 'allocSetOpts2');
+
+				const ret2 = this.resampler.init();
+				NodeAv.FFmpegError.throwIfError(ret2, 'init');
+
+				this.dstFrame = new NodeAv.Frame();
+				this.dstFrame.alloc();
+				this.dstFrame.channelLayout = outLayout;
+				this.dstFrame.sampleRate = this.codecContext.sampleRate;
+				this.dstFrame.format = this.codecContext.sampleFormat;
+				this.dstFrame.nbSamples = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
+				this.dstFrame.duration = BigInt(this.dstFrame.nbSamples);
+				this.dstFrame.allocBuffer();
+
+				this.nextResamplerPts = this.frame.pts;
+			}
+
+			const inputBuffers = this.frame.data;
+			if (!inputBuffers) {
+				throw new DOMException('Frame has no data', 'EncodingError');
+			}
+			await this.resampler.convert(null, 0, inputBuffers, this.frame.nbSamples);
+
+			await this.pullResampledFrames();
+		} else {
+			await this.sendFrameAndReceivePackets(this.frame);
+		}
+	}
+
+	async pullResampledFrames() {
+		assert(this.codecContext);
+		assert(this.resampler);
+		assert(this.dstFrame);
+		assert(this.nextResamplerPts !== null);
+
+		const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
+
+		while (true) {
+			const available = this.resampler.getOutSamples(0);
+			if (available < frameSize) {
+				break;
+			}
+
+			await this.resampler.convert(this.dstFrame.data, frameSize, null, 0);
+
+			this.dstFrame.pts = this.nextResamplerPts;
+
+			await this.sendFrameAndReceivePackets(this.dstFrame);
+			this.nextResamplerPts += BigInt(frameSize);
+		}
+	}
+
+	async sendFrameAndReceivePackets(frame: NodeAv.Frame | null) {
+		assert(this.codecContext);
+
+		const ret = await this.codecContext.sendFrame(frame);
+		NodeAv.FFmpegError.throwIfError(ret, 'Send frame');
+
+		while (true) {
+			const receiveRet = await this.codecContext.receivePacket(this.packet);
+			if (receiveRet === NodeAv.AVERROR_EAGAIN || receiveRet === NodeAv.AVERROR_EOF) {
+				break;
+			}
+
+			this.receivePacket(receiveRet);
+		}
+	}
+
+	receivePacket(ret: number) {
+		assert(this.codecContext);
+		assert(this.firstExpectedTimestamp !== null);
+		NodeAv.FFmpegError.throwIfError(ret, 'Receive packet');
+
+		if (!this.packet.data) {
+			return;
+		}
+
+		let timestamp = Number(this.packet.pts) / this.codecContext.sampleRate;
+		const duration = Number(this.packet.duration) / this.codecContext.sampleRate;
+
+		let data: Uint8Array = this.packet.data;
+
+		let metadata: EncodedAudioChunkMetadata | undefined;
+		if (this.packetEmitted) {
+			metadata = {};
+		} else {
+			// To compensate for any negative timestamp things that FFmpeg might do. It does these for a reason, to
+			// indicate encoder delay, but the notion of this is not yet supported in Mediabunny. Yes, this technically
+			// introduces audio sync drift.
+			this.outputTimestampOffset = Math.max(this.firstExpectedTimestamp - timestamp, 0);
+
+			const codecString = this.config.codec;
+			let description = this.codecContext.extraData
+				? toUint8Array(this.codecContext.extraData)
+				: undefined;
+
+			if (this.codec === 'aac') {
+				if (!description) {
+					throw new Error('Extradata expected for AAC.');
+				}
+
+				// eslint-disable-next-line @stylistic/max-len
+				// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-member-access
+				const isAdts = (this.config as any).aac?.format === 'adts';
+
+				if (isAdts) {
+					const parsedConfig = parseAacAudioSpecificConfig(description);
+					this.adtsHeaderTemplate = buildAdtsHeaderTemplate(parsedConfig);
+					description = undefined; // Not used with 'adts' format
+				}
+			} else if (this.codec === 'opus') {
+				if (!description) {
+					// Technically not required by the WebCodecs/Mediabunny Codec Registry, but we strive to be better
+					throw new Error('Extradata expected for Opus.');
+				}
+			} else if (this.codec === 'vorbis') {
+				if (!description) {
+					throw new Error('Extradata expected for Vorbis.');
+				}
+			} else if (this.codec === 'flac') {
+				if (!description) {
+					throw new Error('Extradata expected for FLAC.');
+				}
+
+				// FFmpeg uses the STREAMINFO block as the extradata, but WebCodecs wants a different format:
+				// 1. The bytes 0x66 0x4C 0x61 0x43 ("fLaC" in ASCII)
+				// 2. A metadata block (called the STREAMINFO block) as described in section 7 of [FLAC]
+				// 3. Other optional metadata blocks (not included here, because, well, they're optional)
+				description = new Uint8Array([
+					0x66, 0x4c, 0x61, 0x43, // 'fLaC'
+					128, 0, 0, description.byteLength,
+					...description,
+				]);
+			}
+
+			metadata = {
+				decoderConfig: {
+					codec: codecString,
+					sampleRate: this.codecContext.sampleRate,
+					numberOfChannels: this.codecContext.channels,
+					description,
+				},
+			};
+		}
+
+		if (this.adtsHeaderTemplate) {
+			const frameLength = data.byteLength + this.adtsHeaderTemplate.header.byteLength;
+			this.adtsHeaderTemplate.bitstream.pos = 30;
+			this.adtsHeaderTemplate.bitstream.writeBits(13, frameLength);
+
+			const final = new Uint8Array(this.adtsHeaderTemplate.header.byteLength + data.byteLength);
+			final.set(this.adtsHeaderTemplate.header, 0);
+			final.set(data, this.adtsHeaderTemplate.header.byteLength);
+
+			data = final;
+		}
+
+		timestamp += this.outputTimestampOffset;
+
+		const packet = new EncodedPacket(
+			data,
+			'key',
+			timestamp,
+			duration,
+		);
+
+		this.packetEmitted = true;
+		this.onPacket(packet, metadata);
+	}
+
+	async flush(): Promise<void> {
+		if (!this.codecContext) {
+			return;
+		}
+
+		outer:
+		if (this.resampler) {
+			assert(this.resamplerInputSampleRate !== null);
+
+			const currentOutSamples = this.resampler.getOutSamples(0);
+			if (currentOutSamples === 0) {
+				break outer; // Clean cut-off point
+			}
+
+			const frameSize = this.codecContext.frameSize || FRAME_SIZE_FALLBACK;
+			assert(currentOutSamples < frameSize); // Because if it's more, it would've already been retrieved
+
+			const inputSamplesNeeded = Math.ceil(
+				((frameSize - currentOutSamples) / this.codecContext.sampleRate) * this.resamplerInputSampleRate,
+			);
+			this.resampler.injectSilence(inputSamplesNeeded);
+
+			await this.pullResampledFrames();
+		}
+
+		await this.sendFrameAndReceivePackets(null);
+
+		this.codecContext.freeContext();
+		this.codecContext = null;
+		this.packetEmitted = false;
+		this.firstExpectedTimestamp = null;
+		this.outputTimestampOffset = 0;
+		this.adtsHeaderTemplate = null;
+
+		this.resampler?.free();
+		this.resampler = null;
+		this.inputParametersKey = null;
+		this.resamplerInputSampleRate = null;
+		this.nextResamplerPts = null;
+		this.dstFrame?.free();
+		this.dstFrame = null;
+	}
+
+	close(): MaybePromise<void> {
+		this.codecContext?.freeContext();
+		this.frame.free();
+		this.packet.free();
+		this.dstFrame?.free();
+		this.resampler?.free();
+	}
+}

package/src/audio-sample.ts

@@ -0,0 +1,101 @@
+/*!
+ * Copyright (c) 2026-present, Vanilagy and contributors
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+import { AudioSample, AudioSampleResource } from 'mediabunny';
+import * as NodeAv from 'node-av';
+import { fromAudioSampleFormat, getChannelLayout, toAudioSampleFormat } from './misc';
+import { assert, toUint8Array } from '../../../src/misc';
+
+/**
+ * A custom `AudioSampleResource` backed by NodeAV's
+ * [`Frame`](https://seydx.github.io/node-av/api/lib/classes/Frame.html), which in turn is backed by FFmpeg's
+ * [`AVFrame`](https://ffmpeg.org/doxygen/2.7/structAVFrame.html). You can use this resource to create `AudioSample`
+ * instances that are directly backed by FFmpeg's `AVFrame` without data having to be copied.
+ *
+ * When passed, the `Frame` is now owned by resource, meaning it takes care of closing the frame later. If you want to
+ * keep a copy for your own use, clone the frame first.
+ *
+ * @group \@mediabunny/server
+ * @public
+ */
+export class AvFrameAudioSampleResource extends AudioSampleResource {
+	/** @internal */
+	_frame: NodeAv.Frame | null;
+
+	/**
+	 * The NodeAV [`Frame`](https://seydx.github.io/node-av/api/lib/classes/Frame.html) instance backing this resource.
+	 * Access throws if the resource has already been closed.
+	 */
+	get frame() {
+		if (!this._frame) {
+			throw new Error('AvFrameAudioSampleResource has been closed.');
+		}
+
+		return this._frame;
+	}
+
+	constructor(frame: NodeAv.Frame) {
+		super();
+
+		if (frame.getMediaType() !== NodeAv.AVMEDIA_TYPE_AUDIO) {
+			throw new Error('AvFrameAudioSampleResource must be initialized with an audio frame.');
+		}
+
+		this._frame = frame;
+	}
+
+	getFormat(): AudioSampleFormat {
+		const result = toAudioSampleFormat(this.frame.format as NodeAv.AVSampleFormat);
+		if (result === null) {
+			const name = NodeAv.avGetSampleFmtName(this.frame.format as NodeAv.AVSampleFormat);
+			throw new TypeError(`Unsupported audio sample format: ${name} (${this.frame.format})`);
+		}
+
+		return result;
+	}
+
+	getSampleRate(): number {
+		return this.frame.sampleRate;
+	}
+
+	getNumberOfChannels(): number {
+		return this.frame.channels;
+	}
+
+	getNumberOfFrames(): number {
+		return this.frame.nbSamples;
+	}
+
+	getTimestamp(): number {
+		return Number(this.frame.pts) / this.frame.timeBase.den;
+	}
+
+	close(): void {
+		this.frame.free();
+		this._frame = null;
+	}
+
+	getDataPlane(planeIndex: number): Uint8Array {
+		assert(this.frame.data && planeIndex < this.frame.data.length);
+		return toUint8Array(this.frame.data[planeIndex]!);
+	}
+}
+
+export const copyAudioSampleToAvFrame = (sample: AudioSample, frame: NodeAv.Frame) => {
+	frame.format = fromAudioSampleFormat(sample.format);
+	frame.nbSamples = sample.numberOfFrames;
+	frame.sampleRate = sample.sampleRate;
+	frame.channelLayout = getChannelLayout(sample.numberOfChannels);
+
+	frame.allocBuffer();
+	assert(frame.data);
+
+	for (let i = 0; i < frame.data.length; i++) {
+		sample.copyTo(frame.data[i]!, { planeIndex: i });
+	}
+};