@remotion/media-parser 4.0.194 → 4.0.196

package/src/boxes/iso-base-media/tkhd.ts

@@ -16,8 +16,47 @@ export interface TkhdBox extends BaseBox {
 	matrix: ThreeDMatrix;
 	width: number;
 	height: number;
+	unrotatedWidth: number;
+	unrotatedHeight: number;
+	rotation: number;
 }
 
+type Matrix2x2 = readonly [number, number, number, number];
+
+function getRotationAngleFromMatrix(matrix: Matrix2x2): number {
+	// Extract elements from the matrix
+	const [a, b, c, d] = matrix;
+
+	// Check if the matrix is a valid rotation matrix
+	if (Math.round(a * a + b * b) !== 1 || Math.round(c * c + d * d) !== 1) {
+		throw new Error('The provided matrix is not a valid rotation matrix.');
+	}
+
+	// Calculate the angle using the atan2 function
+	const angleRadians = Math.atan2(c, a); // atan2(sin(θ), cos(θ))
+	const angleDegrees = angleRadians * (180 / Math.PI); // Convert radians to degrees
+
+	return angleDegrees;
+}
+
+const applyRotation = ({
+	matrix,
+	width,
+	height,
+}: {
+	matrix: Matrix2x2;
+	width: number;
+	height: number;
+}) => {
+	const newWidth = matrix[0] * width + matrix[1] * height; // 0*3840 + 1*2160
+	const newHeight = matrix[2] * width + matrix[3] * height; // -1*3840 + 0*2160
+
+	return {
+		width: Math.abs(newWidth),
+		height: Math.abs(newHeight),
+	};
+};
+
 export const parseTkhd = ({
 	iterator,
 	offset,
@@ -66,25 +105,30 @@ export const parseTkhd = ({
 	iterator.discard(2);
 
 	const matrix = [
-		iterator.getUint32(),
-		iterator.getUint32(),
-		iterator.getUint32(),
-		iterator.getUint32(),
-		iterator.getUint32(),
-		iterator.getUint32(),
-		iterator.getUint32(),
-		iterator.getUint32(),
-		iterator.getUint32(),
+		iterator.getFixedPointSigned1616Number(),
+		iterator.getFixedPointSigned1616Number(),
+		iterator.getFixedPointSigned230Number(),
+		iterator.getFixedPointSigned1616Number(),
+		iterator.getFixedPointSigned1616Number(),
+		iterator.getFixedPointSigned230Number(),
+		iterator.getFixedPointSigned1616Number(),
+		iterator.getFixedPointSigned1616Number(),
+		iterator.getFixedPointSigned230Number(),
 	];
+
+	const rotationMatrix = [matrix[0], matrix[1], matrix[3], matrix[4]] as const;
+
 	const widthWithoutRotationApplied =
-		iterator.getUint32() / (matrix[0] === 0 ? 1 : matrix[0]);
-	const heightWithoutRotationApplied =
-		iterator.getUint32() / (matrix[4] === 0 ? 1 : matrix[4]);
+		iterator.getFixedPointUnsigned1616Number();
+	const heightWithoutRotationApplied = iterator.getFixedPointSigned1616Number();
+
+	const {width, height} = applyRotation({
+		matrix: rotationMatrix,
+		width: widthWithoutRotationApplied,
+		height: heightWithoutRotationApplied,
+	});
 
-	// TODO: This is not correct, HEVC videos with matrix is wrong
-	const width = widthWithoutRotationApplied / (matrix[1] === 0 ? 1 : matrix[1]);
-	const height =
-		heightWithoutRotationApplied / (matrix[1] === 0 ? 1 : matrix[1]);
+	const rotation = getRotationAngleFromMatrix(rotationMatrix);
 
 	return {
 		offset,
@@ -101,5 +145,8 @@ export const parseTkhd = ({
 		width,
 		height,
 		version,
+		rotation,
+		unrotatedWidth: widthWithoutRotationApplied,
+		unrotatedHeight: heightWithoutRotationApplied,
 	};
 };

package/src/boxes/iso-base-media/trak/trak.ts

@@ -1,5 +1,6 @@
 import type {BufferIterator} from '../../../buffer-iterator';
 import type {AnySegment} from '../../../parse-result';
+import type {ParserContext} from '../../../parser-context';
 import type {BaseBox} from '../base-type';
 import {parseBoxes} from '../process-box';
 
@@ -8,20 +9,24 @@ export interface TrakBox extends BaseBox {
 	children: AnySegment[];
 }
 
-export const parseTrak = ({
+export const parseTrak = async ({
 	data,
 	size,
 	offsetAtStart,
+	options,
 }: {
 	data: BufferIterator;
 	size: number;
 	offsetAtStart: number;
-}): TrakBox => {
-	const children = parseBoxes({
+	options: ParserContext;
+}): Promise<TrakBox> => {
+	const children = await parseBoxes({
 		iterator: data,
 		maxBytes: size - (data.counter.getOffset() - offsetAtStart),
 		allowIncompleteBoxes: false,
 		initialBoxes: [],
+		options,
+		continueMdat: false,
 	});
 
 	if (children.status === 'incomplete') {

package/src/boxes/iso-base-media/void-box.ts

@@ -0,0 +1,4 @@
+export interface VoidBox {
+	type: 'void-box';
+	boxSize: number;
+}

package/src/boxes/webm/allowed-partial-segments.ts

@@ -0,0 +1 @@
+export const allowedPartialSegments = ['0x1f43b675', '0x18538067'];

package/src/boxes/webm/av1-codec-private.ts

@@ -0,0 +1,113 @@
+import {getArrayBufferIterator} from '../../buffer-iterator';
+import type {ColorParameterBox} from '../iso-base-media/stsd/colr';
+
+export const parseAv1PrivateData = (
+	data: Uint8Array,
+	colrAtom: ColorParameterBox | null,
+) => {
+	const iterator = getArrayBufferIterator(data);
+	iterator.startReadingBits();
+	if (iterator.getBits(1) !== 1) {
+		iterator.destroy();
+		throw new Error('Expected av1 private data to be version 1');
+	}
+
+	const version = iterator.getBits(7);
+	if (version !== 1) {
+		iterator.destroy();
+		throw new Error(
+			`Expected av1 private data to be version 1, got ${version}`,
+		);
+	}
+
+	let str = 'av01.';
+
+	// https://aomediacodec.github.io/av1-isobmff/#codecsparam
+
+	const seqProfile = iterator.getBits(3);
+	// Profile
+	str += seqProfile;
+	str += '.';
+
+	const seq_level_idx = iterator.getBits(5);
+	const seq_tier_0 = iterator.getBits(1);
+	// Level
+	// The level parameter value SHALL equal the first level value indicated by seq_level_idx in the Sequence Header OBU
+	str += seq_level_idx.toString(16).padStart(2, '0');
+	str += seq_tier_0 ? 'H' : 'M';
+	str += '.';
+
+	// bitDepth
+	// The bitDepth parameter value SHALL equal the value of BitDepth variable as defined in [AV1] derived from the Sequence Header OBU
+	const high_bitdepth = iterator.getBits(1);
+	const twelve_bit = iterator.getBits(1);
+	const bitDepth =
+		high_bitdepth && seqProfile === 2
+			? twelve_bit
+				? 12
+				: 10
+			: high_bitdepth
+				? 10
+				: 8;
+	str += bitDepth.toString().padStart(2, '0');
+	str += '.';
+
+	// monochrome
+	// The monochrome parameter value, represented by a single digit decimal, SHALL equal the value of mono_chrome in the Sequence Header OBU
+	const mono_chrome = iterator.getBits(1);
+	str += mono_chrome ? '1' : '0';
+	str += '.';
+
+	// The chromaSubsampling parameter value, represented by a three-digit decimal,
+	// SHALL have its first digit equal to subsampling_x
+	const subsampling_x = iterator.getBits(1);
+	str += subsampling_x ? '1' : '0';
+
+	// and its second digit equal to subsampling_y.
+	const subsampling_y = iterator.getBits(1);
+	str += subsampling_y ? '1' : '0';
+
+	// If both subsampling_x and subsampling_y are set to 1, then the third digit SHALL be equal to chroma_sample_position, otherwise it SHALL be set to 0
+	const chroma_sample_position = iterator.getBits(2);
+	str +=
+		subsampling_x && subsampling_y
+			? chroma_sample_position === 1
+				? '1'
+				: '0'
+			: '0';
+	str += '.';
+
+	if (colrAtom) {
+		str += colrAtom.primaries.toString().padStart(2, '0');
+		str += '.';
+		str += colrAtom.transfer.toString().padStart(2, '0');
+		str += '.';
+		str += colrAtom.matrixIndex.toString().padStart(2, '0');
+		str += '.';
+		str += colrAtom.fullRangeFlag ? '1' : '0';
+	} else {
+		// Otherwise, the color_description_present_flag is set to 0 in the Sequence Header OBU. The colorPrimaries, transferCharacteristics, and matrixCoefficients parameter values SHOULD be set to the default values below.
+		// colorPrimaries 01 (ITU-R BT.709)
+		str += '01';
+		str += '.';
+		// transferCharacteristics 01 (ITU-R BT.709)
+		str += '01';
+		str += '.';
+		// matrixCoefficients 00 (ITU-R BT.709)
+		str += '01';
+		str += '.';
+		// videoFullRangeFlag 0 (studio swing representation)
+		str += '0';
+	}
+
+	// If the codecs parameter string ends with ".0.110.01.01.01.0" (containing all the default values below), that trailing part of the string SHOULD be omitted.
+	const suffix = '.0.110.01.01.01.0';
+
+	if (str.endsWith(suffix)) {
+		str = str.slice(0, -suffix.length);
+	}
+
+	iterator.destroy();
+
+	return str;
+};

package/src/boxes/webm/description.ts

@@ -0,0 +1,101 @@
+import {getArrayBufferIterator} from '../../buffer-iterator';
+import {getCodecSegment, getPrivateData} from '../../traversal';
+import type {TrackEntrySegment} from './segments/track-entry';
+
+export const getAudioDescription = (
+	track: TrackEntrySegment,
+): undefined | Uint8Array => {
+	const codec = getCodecSegment(track);
+	if (!codec || codec.codec !== 'A_VORBIS') {
+		return undefined;
+	}
+
+	// how to parse vorbis private
+	// https://github.com/google/ExoPlayer/blob/dd430f7053a1a3958deea3ead6a0565150c06bfc/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mkv/MatroskaExtractor.java#L2466
+	const privateData = getPrivateData(track);
+	if (!privateData) {
+		return undefined;
+	}
+
+	if (privateData[0] !== 2) {
+		throw new Error('Expected vorbis private data version 2');
+	}
+
+	let offset = 1;
+	let vorbisInfoLength = 0;
+	let vorbisSkipLength = 0;
+
+	while ((privateData[offset] & 0xff) === 0xff) {
+		vorbisInfoLength += 0xff;
+		offset++;
+	}
+
+	vorbisInfoLength += privateData[offset++] & 0xff;
+
+	while ((privateData[offset] & 0xff) === 0xff) {
+		vorbisSkipLength += 0xff;
+		offset++;
+	}
+
+	vorbisSkipLength += privateData[offset++] & 0xff;
+
+	if (privateData[offset] !== 0x01) {
+		throw new Error('Error parsing vorbis codec private');
+	}
+
+	const vorbisInfo = privateData.slice(offset, offset + vorbisInfoLength);
+	offset += vorbisInfoLength;
+
+	if (privateData[offset] !== 0x03) {
+		throw new Error('Error parsing vorbis codec private');
+	}
+
+	const vorbisComments = privateData.slice(offset, offset + vorbisSkipLength);
+	offset += vorbisSkipLength;
+
+	if (privateData[offset] !== 0x05) {
+		throw new Error('Error parsing vorbis codec private');
+	}
+
+	const vorbisBooks = privateData.slice(offset);
+
+	const bufferIterator = getArrayBufferIterator(vorbisInfo.slice(0));
+
+	// type
+	bufferIterator.getUint8();
+	// vorbis
+	const vorbis = bufferIterator.getByteString(6);
+	if (vorbis !== 'vorbis') {
+		throw new Error('Error parsing vorbis codec private');
+	}
+
+	const vorbisVersion = bufferIterator.getUint32Le();
+	if (vorbisVersion !== 0) {
+		throw new Error('Error parsing vorbis codec private');
+	}
+
+	const vorbisDescription = new Uint8Array([
+		// constructing the vorbis description
+		// This format consists in the page_segments field
+		/**
+		 * The number of segment entries to appear in the segment table.
+		 * The maximum number of 255 segments (255 bytes each) sets the maximum possible physical page size at 65307 bytes or just under 64kB (thus we know that a header corrupted so as destroy sizing/alignment information will not cause a runaway bitstream.
+		 * We'll read in the page according to the corrupted size information that's guaranteed to be a reasonable size regardless, notice the checksum mismatch, drop sync and then look for recapture).
+		 */
+		2,
+		// followed by the segment_table field
+		// offset of the comments table
+		vorbisInfo.length,
+		// offset of the codebooks table
+		vorbisComments.length,
+		// followed by the three Vorbis header packets,
+		// respectively the identification header,
+		...vorbisInfo,
+		// the comments header,
+		...vorbisComments,
+		// and the setup header, in this order, as described in section 4.2 of [VORBIS].
+		...vorbisBooks,
+	]);
+
+	return vorbisDescription;
+};

package/src/boxes/webm/get-ready-tracks.ts

@@ -0,0 +1,36 @@
+import type {Track} from '../../get-tracks';
+import {getTracksSegment} from '../../traversal';
+import {getTrack} from './get-track';
+import type {MainSegment} from './segments/main';
+
+export const getTracksFromMatroska = (
+	segment: MainSegment,
+	timescale: number,
+): Track[] => {
+	const tracksSegment = getTracksSegment(segment);
+	if (!tracksSegment) {
+		throw new Error('No tracks segment');
+	}
+
+	const tracks: Track[] = [];
+
+	for (const trackEntrySegment of tracksSegment.children) {
+		if (trackEntrySegment.type === 'crc32-segment') {
+			continue;
+		}
+
+		if (trackEntrySegment.type !== 'track-entry-segment') {
+			throw new Error('Expected track entry segment');
+		}
+
+		const track = getTrack({
+			track: trackEntrySegment,
+			timescale,
+		});
+		if (track) {
+			tracks.push(track);
+		}
+	}
+
+	return tracks;
+};

package/src/boxes/webm/get-track.ts

@@ -0,0 +1,250 @@
+import {getArrayBufferIterator} from '../../buffer-iterator';
+import type {AudioTrack, VideoTrack} from '../../get-tracks';
+import {getHvc1CodecString} from '../../make-hvc1-codec-strings';
+import {
+	getBitDepth,
+	getCodecSegment,
+	getDisplayHeightSegment,
+	getDisplayWidthSegment,
+	getHeightSegment,
+	getNumberOfChannels,
+	getPrivateData,
+	getSampleRate,
+	getTrackId,
+	getTrackTypeSegment,
+	getWidthSegment,
+} from '../../traversal';
+import {parseAv1PrivateData} from './av1-codec-private';
+import {getAudioDescription} from './description';
+import type {CodecSegment, TrackEntrySegment} from './segments/track-entry';
+
+const getDescription = (track: TrackEntrySegment): undefined | Uint8Array => {
+	const codec = getCodecSegment(track);
+	if (!codec) {
+		return undefined;
+	}
+
+	if (codec.codec === 'V_MPEG4/ISO/AVC' || codec.codec === 'V_MPEGH/ISO/HEVC') {
+		const priv = getPrivateData(track);
+		if (priv) {
+			return priv;
+		}
+	}
+
+	return undefined;
+};
+
+const getMatroskaVideoCodecString = ({
+	track,
+	codecSegment: codec,
+}: {
+	track: TrackEntrySegment;
+	codecSegment: CodecSegment;
+}): string | null => {
+	if (codec.codec === 'V_VP8') {
+		return 'vp8';
+	}
+
+	if (codec.codec === 'V_VP9') {
+		const priv = getPrivateData(track);
+		if (priv) {
+			throw new Error(
+				'@remotion/media-parser cannot handle the private data for VP9. Do you have an example file you could send so we can implement it?',
+			);
+		}
+
+		return 'vp09.00.10.08';
+	}
+
+	if (codec.codec === 'V_MPEG4/ISO/AVC') {
+		const priv = getPrivateData(track);
+		if (priv) {
+			return `avc1.${priv[1].toString(16).padStart(2, '0')}${priv[2].toString(16).padStart(2, '0')}${priv[3].toString(16).padStart(2, '0')}`;
+		}
+
+		throw new Error('Could not find a CodecPrivate field in TrackEntry');
+	}
+
+	if (codec.codec === 'V_AV1') {
+		const priv = getPrivateData(track);
+
+		if (!priv) {
+			throw new Error('Expected private data in AV1 track');
+		}
+
+		return parseAv1PrivateData(priv, null);
+	}
+
+	if (codec.codec === 'V_MPEGH/ISO/HEVC') {
+		const priv = getPrivateData(track);
+		const iterator = getArrayBufferIterator(priv as Uint8Array);
+
+		return 'hvc1.' + getHvc1CodecString(iterator);
+	}
+
+	throw new Error(`Unknown codec: ${codec.codec}`);
+};
+
+const getMatroskaAudioCodecString = (track: TrackEntrySegment): string => {
+	const codec = getCodecSegment(track);
+	if (!codec) {
+		throw new Error('Expected codec segment');
+	}
+
+	if (codec.codec === 'A_OPUS') {
+		return 'opus';
+	}
+
+	if (codec.codec === 'A_VORBIS') {
+		return 'vorbis';
+	}
+
+	if (codec.codec === 'A_PCM/INT/LIT') {
+		// https://github.com/ietf-wg-cellar/matroska-specification/issues/142#issuecomment-330004950
+		// Audio samples MUST be considered as signed values, except if the audio bit depth is 8 which MUST be interpreted as unsigned values.
+
+		const bitDepth = getBitDepth(track);
+		if (bitDepth === null) {
+			throw new Error('Expected bit depth');
+		}
+
+		if (bitDepth === 8) {
+			return 'pcm-u8';
+		}
+
+		return 'pcm-s' + bitDepth;
+	}
+
+	if (codec.codec === 'A_AAC') {
+		const priv = getPrivateData(track);
+
+		const iterator = getArrayBufferIterator(priv as Uint8Array);
+
+		iterator.startReadingBits();
+		/**
+		 * ChatGPT
+		 * 	▪	The first 5 bits represent the AOT.
+				▪	Common values:
+				◦	1 for AAC Main
+				◦	2 for AAC LC (Low Complexity)
+				◦	3 for AAC SSR (Scalable Sample Rate)
+				◦	4 for AAC LTP (Long Term Prediction)
+				◦	5 for SBR (Spectral Band Replication)
+				◦	29 for HE-AAC (which uses SBR with AAC LC)
+		 */
+		/** 
+		 * Fully qualified codec: 
+		 * This codec has multiple possible codec strings:
+			"mp4a.40.2" — MPEG-4 AAC LC
+			"mp4a.40.02" — MPEG-4 AAC LC, leading 0 for Aud-OTI compatibility
+			"mp4a.40.5" — MPEG-4 HE-AAC v1 (AAC LC + SBR)
+			"mp4a.40.05" — MPEG-4 HE-AAC v1 (AAC LC + SBR), leading 0 for Aud-OTI compatibility
+			"mp4a.40.29" — MPEG-4 HE-AAC v2 (AAC LC + SBR + PS)
+			"mp4a.67" — MPEG-2 AAC LC
+		*/
+
+		const profile = iterator.getBits(5);
+		iterator.stopReadingBits();
+		iterator.destroy();
+
+		return `mp4a.40.${profile.toString().padStart(2, '0')}`;
+	}
+
+	if (codec.codec === 'A_MPEG/L3') {
+		return 'mp3';
+	}
+
+	throw new Error(`Unknown codec: ${codec.codec}`);
+};
+
+export const getTrack = ({
+	timescale,
+	track,
+}: {
+	timescale: number;
+	track: TrackEntrySegment;
+}): VideoTrack | AudioTrack | null => {
+	const trackType = getTrackTypeSegment(track);
+
+	if (!trackType) {
+		throw new Error('Expected track type segment');
+	}
+
+	const trackId = getTrackId(track);
+
+	if (trackType.trackType === 'video') {
+		const width = getWidthSegment(track);
+
+		if (width === null) {
+			throw new Error('Expected width segment');
+		}
+
+		const height = getHeightSegment(track);
+
+		if (height === null) {
+			throw new Error('Expected height segment');
+		}
+
+		const displayHeight = getDisplayHeightSegment(track);
+		const displayWidth = getDisplayWidthSegment(track);
+
+		const codec = getCodecSegment(track);
+		if (!codec) {
+			return null;
+		}
+
+		const codecString = getMatroskaVideoCodecString({
+			track,
+			codecSegment: codec,
+		});
+
+		if (!codecString) {
+			return null;
+		}
+
+		return {
+			type: 'video',
+			trackId,
+			codec: codecString,
+			description: getDescription(track),
+			height: displayHeight ? displayHeight.displayHeight : height.height,
+			width: displayWidth ? displayWidth.displayWidth : width.width,
+			sampleAspectRatio: {
+				numerator: 1,
+				denominator: 1,
+			},
+			timescale,
+			samplePositions: [],
+			codedHeight: height.height,
+			codedWidth: width.width,
+			displayAspectHeight: displayHeight
+				? displayHeight.displayHeight
+				: height.height,
+			displayAspectWidth: displayWidth
+				? displayWidth.displayWidth
+				: width.width,
+			rotation: 0,
+		};
+	}
+
+	if (trackType.trackType === 'audio') {
+		const sampleRate = getSampleRate(track);
+		const numberOfChannels = getNumberOfChannels(track);
+		if (sampleRate === null) {
+			throw new Error('Could not find sample rate or number of channels');
+		}
+
+		return {
+			type: 'audio',
+			trackId,
+			codec: getMatroskaAudioCodecString(track),
+			samplePositions: null,
+			timescale,
+			numberOfChannels,
+			sampleRate,
+			description: getAudioDescription(track),
+		};
+	}
+
+	return null;
+};

package/src/boxes/webm/parse-webm-header.ts

@@ -1,9 +1,13 @@
 import type {BufferIterator} from '../../buffer-iterator';
 import type {ParseResult} from '../../parse-result';
+import type {ParserContext} from '../../parser-context';
 import {expectSegment} from './segments';
 
 // Parsing according to https://darkcoding.net/software/reading-mediarecorders-webm-opus-output/
-export const parseWebm = (counter: BufferIterator): ParseResult => {
+export const parseWebm = (
+	counter: BufferIterator,
+	parserContext: ParserContext,
+): Promise<ParseResult> => {
 	counter.discard(4);
 	const length = counter.getVint();
 
@@ -14,5 +18,5 @@ export const parseWebm = (counter: BufferIterator): ParseResult => {
 	// Discard header for now
 	counter.discard(length);
 
-	return {status: 'done', segments: [expectSegment(counter)]};
+	return expectSegment(counter, parserContext);
 };