@remotion/media-parser 4.0.201 → 4.0.202

package/src/boxes/webm/get-track.ts

@@ -16,15 +16,16 @@ import {
 } from '../../traversal';
 import {parseAv1PrivateData} from './av1-codec-private';
 import {getAudioDescription} from './description';
-import type {CodecSegment, TrackEntrySegment} from './segments/track-entry';
+import type {CodecIdSegment, TrackEntry} from './segments/all-segments';
+import {trackTypeToString} from './segments/track-entry';
 
-const getDescription = (track: TrackEntrySegment): undefined | Uint8Array => {
+const getDescription = (track: TrackEntry): undefined | Uint8Array => {
 	const codec = getCodecSegment(track);
 	if (!codec) {
 		return undefined;
 	}
 
-	if (codec.codec === 'V_MPEG4/ISO/AVC' || codec.codec === 'V_MPEGH/ISO/HEVC') {
+	if (codec.value === 'V_MPEG4/ISO/AVC' || codec.value === 'V_MPEGH/ISO/HEVC') {
 		const priv = getPrivateData(track);
 		if (priv) {
 			return priv;
@@ -38,14 +39,14 @@ const getMatroskaVideoCodecString = ({
 	track,
 	codecSegment: codec,
 }: {
-	track: TrackEntrySegment;
-	codecSegment: CodecSegment;
+	track: TrackEntry;
+	codecSegment: CodecIdSegment;
 }): string | null => {
-	if (codec.codec === 'V_VP8') {
+	if (codec.value === 'V_VP8') {
 		return 'vp8';
 	}
 
-	if (codec.codec === 'V_VP9') {
+	if (codec.value === 'V_VP9') {
 		const priv = getPrivateData(track);
 		if (priv) {
 			throw new Error(
@@ -56,7 +57,7 @@ const getMatroskaVideoCodecString = ({
 		return 'vp09.00.10.08';
 	}
 
-	if (codec.codec === 'V_MPEG4/ISO/AVC') {
+	if (codec.value === 'V_MPEG4/ISO/AVC') {
 		const priv = getPrivateData(track);
 		if (priv) {
 			return `avc1.${priv[1].toString(16).padStart(2, '0')}${priv[2].toString(16).padStart(2, '0')}${priv[3].toString(16).padStart(2, '0')}`;
@@ -65,7 +66,7 @@ const getMatroskaVideoCodecString = ({
 		throw new Error('Could not find a CodecPrivate field in TrackEntry');
 	}
 
-	if (codec.codec === 'V_AV1') {
+	if (codec.value === 'V_AV1') {
 		const priv = getPrivateData(track);
 
 		if (!priv) {
@@ -75,31 +76,34 @@ const getMatroskaVideoCodecString = ({
 		return parseAv1PrivateData(priv, null);
 	}
 
-	if (codec.codec === 'V_MPEGH/ISO/HEVC') {
+	if (codec.value === 'V_MPEGH/ISO/HEVC') {
 		const priv = getPrivateData(track);
-		const iterator = getArrayBufferIterator(priv as Uint8Array);
+		const iterator = getArrayBufferIterator(
+			priv as Uint8Array,
+			(priv as Uint8Array).length,
+		);
 
 		return 'hvc1.' + getHvc1CodecString(iterator);
 	}
 
-	throw new Error(`Unknown codec: ${codec.codec}`);
+	throw new Error(`Unknown codec: ${codec.value}`);
 };
 
-const getMatroskaAudioCodecString = (track: TrackEntrySegment): string => {
+const getMatroskaAudioCodecString = (track: TrackEntry): string => {
 	const codec = getCodecSegment(track);
 	if (!codec) {
 		throw new Error('Expected codec segment');
 	}
 
-	if (codec.codec === 'A_OPUS') {
+	if (codec.value === 'A_OPUS') {
 		return 'opus';
 	}
 
-	if (codec.codec === 'A_VORBIS') {
+	if (codec.value === 'A_VORBIS') {
 		return 'vorbis';
 	}
 
-	if (codec.codec === 'A_PCM/INT/LIT') {
+	if (codec.value === 'A_PCM/INT/LIT') {
 		// https://github.com/ietf-wg-cellar/matroska-specification/issues/142#issuecomment-330004950
 		// Audio samples MUST be considered as signed values, except if the audio bit depth is 8 which MUST be interpreted as unsigned values.
 
@@ -115,10 +119,13 @@ const getMatroskaAudioCodecString = (track: TrackEntrySegment): string => {
 		return 'pcm-s' + bitDepth;
 	}
 
-	if (codec.codec === 'A_AAC') {
+	if (codec.value === 'A_AAC') {
 		const priv = getPrivateData(track);
 
-		const iterator = getArrayBufferIterator(priv as Uint8Array);
+		const iterator = getArrayBufferIterator(
+			priv as Uint8Array,
+			(priv as Uint8Array).length,
+		);
 
 		iterator.startReadingBits();
 		/**
@@ -150,11 +157,11 @@ const getMatroskaAudioCodecString = (track: TrackEntrySegment): string => {
 		return `mp4a.40.${profile.toString().padStart(2, '0')}`;
 	}
 
-	if (codec.codec === 'A_MPEG/L3') {
+	if (codec.value === 'A_MPEG/L3') {
 		return 'mp3';
 	}
 
-	throw new Error(`Unknown codec: ${codec.codec}`);
+	throw new Error(`Unknown codec: ${codec.value}`);
 };
 
 export const getTrack = ({
@@ -162,7 +169,7 @@ export const getTrack = ({
 	track,
 }: {
 	timescale: number;
-	track: TrackEntrySegment;
+	track: TrackEntry;
 }): VideoTrack | AudioTrack | null => {
 	const trackType = getTrackTypeSegment(track);
 
@@ -172,7 +179,7 @@ export const getTrack = ({
 
 	const trackId = getTrackId(track);
 
-	if (trackType.trackType === 'video') {
+	if (trackTypeToString(trackType.value.value) === 'video') {
 		const width = getWidthSegment(track);
 
 		if (width === null) {
@@ -207,27 +214,27 @@ export const getTrack = ({
 			trackId,
 			codec: codecString,
 			description: getDescription(track),
-			height: displayHeight ? displayHeight.displayHeight : height.height,
-			width: displayWidth ? displayWidth.displayWidth : width.width,
+			height: displayHeight ? displayHeight.value.value : height.value.value,
+			width: displayWidth ? displayWidth.value.value : width.value.value,
 			sampleAspectRatio: {
 				numerator: 1,
 				denominator: 1,
 			},
 			timescale,
 			samplePositions: [],
-			codedHeight: height.height,
-			codedWidth: width.width,
+			codedHeight: height.value.value,
+			codedWidth: width.value.value,
 			displayAspectHeight: displayHeight
-				? displayHeight.displayHeight
-				: height.height,
+				? displayHeight.value.value
+				: height.value.value,
 			displayAspectWidth: displayWidth
-				? displayWidth.displayWidth
-				: width.width,
+				? displayWidth.value.value
+				: width.value.value,
 			rotation: 0,
 		};
 	}
 
-	if (trackType.trackType === 'audio') {
+	if (trackTypeToString(trackType.value.value) === 'audio') {
 		const sampleRate = getSampleRate(track);
 		const numberOfChannels = getNumberOfChannels(track);
 		if (sampleRate === null) {

package/src/boxes/webm/make-header.ts

@@ -1,15 +1,12 @@
 import {getVariableInt} from './ebml';
 import type {
-	Ebml,
-	EbmlWithChildren,
-	EbmlWithHexString,
-	EbmlWithString,
-	EbmlWithUint8,
-	EmblTypes,
-	HeaderStructure,
+	FloatWithSize,
+	PossibleEbml,
+	PossibleEbmlOrUint8Array,
+	UintWithSize,
 	matroskaElements,
 } from './segments/all-segments';
-import {getIdForName} from './segments/all-segments';
+import {ebmlMap, getIdForName} from './segments/all-segments';
 
 export const webmPattern = new Uint8Array([0x1a, 0x45, 0xdf, 0xa3]);
 
@@ -26,24 +23,7 @@ const matroskaToHex = (
 	return numbers;
 };
 
-type Numbers = '0' | '1' | '2' | '3' | '4' | '5' | '6';
-
-type ChildFields<Struct extends HeaderStructure> = {
-	[key in keyof Struct &
-		Numbers as Struct[key]['name']]: EmblTypes[Struct[key]['type']];
-};
-
-type SerializeValue<Struct extends Ebml> = Struct extends EbmlWithChildren
-	? ChildFields<Struct['children']>
-	: Struct extends EbmlWithString
-		? string
-		: Struct extends EbmlWithUint8
-			? number
-			: Struct extends EbmlWithHexString
-				? string
-				: undefined;
-
-function putUintDynamic(number: number) {
+function putUintDynamic(number: number, minimumLength: number | null) {
 	if (number < 0) {
 		throw new Error(
 			'This function is designed for non-negative integers only.',
@@ -51,7 +31,10 @@ function putUintDynamic(number: number) {
 	}
 
 	// Calculate the minimum number of bytes needed to store the integer
-	const length = Math.ceil(Math.log2(number + 1) / 8);
+	const length = Math.max(
+		minimumLength ?? 0,
+		Math.ceil(Math.log2(number + 1) / 8),
+	);
 	const bytes = new Uint8Array(length);
 
 	for (let i = 0; i < length; i++) {
@@ -62,36 +45,42 @@ function putUintDynamic(number: number) {
 	return bytes;
 }
 
-const makeFromHeaderStructure = <Struct extends Ebml>(
-	struct: Struct,
-	fields: SerializeValue<Struct>,
+const makeFromHeaderStructure = (
+	fields: PossibleEbmlOrUint8Array,
 ): Uint8Array => {
+	if (fields instanceof Uint8Array) {
+		return fields;
+	}
+
 	const arrays: Uint8Array[] = [];
 
+	const struct = ebmlMap[getIdForName(fields.type)];
+
+	if (struct.type === 'uint8array') {
+		return fields.value as Uint8Array;
+	}
+
 	if (struct.type === 'children') {
-		for (const item of struct.children) {
-			arrays.push(
-				makeMatroskaHeader(
-					item,
-					// @ts-expect-error
-					fields[item.name],
-				),
-			);
+		for (const item of fields.value as PossibleEbml[]) {
+			arrays.push(makeMatroskaBytes(item));
 		}
 
 		return combineUint8Arrays(arrays);
 	}
 
 	if (struct.type === 'string') {
-		return new TextEncoder().encode(fields as string);
+		return new TextEncoder().encode(fields.value as string);
 	}
 
 	if (struct.type === 'uint') {
-		return putUintDynamic(fields as number);
+		return putUintDynamic(
+			(fields.value as UintWithSize).value,
+			(fields.value as UintWithSize).byteLength,
+		);
 	}
 
 	if (struct.type === 'hex-string') {
-		const hex = (fields as string).substring(2);
+		const hex = (fields.value as string).substring(2);
 		const arr = new Uint8Array(hex.length / 2);
 		for (let i = 0; i < hex.length; i += 2) {
 			const byte = parseInt(hex.substring(i, i + 2), 16);
@@ -101,27 +90,45 @@ const makeFromHeaderStructure = <Struct extends Ebml>(
 		return arr;
 	}
 
-	if (struct.type === 'void') {
-		throw new Error('Serializing Void is not implemented');
+	if (struct.type === 'float') {
+		const value = fields.value as FloatWithSize;
+		if (value.size === '32') {
+			const dataView = new DataView(new ArrayBuffer(4));
+			dataView.setFloat32(0, value.value);
+			return new Uint8Array(dataView.buffer);
+		}
+
+		const dataView2 = new DataView(new ArrayBuffer(8));
+		dataView2.setFloat64(0, value.value);
+		return new Uint8Array(dataView2.buffer);
 	}
 
 	throw new Error('Unexpected type');
 };
 
-export const makeMatroskaHeader = <Struct extends Ebml>(
-	struct: Struct,
-	fields: SerializeValue<Struct>,
-) => {
-	const value = makeFromHeaderStructure(struct, fields);
+export const makeMatroskaBytes = (fields: PossibleEbmlOrUint8Array) => {
+	if (fields instanceof Uint8Array) {
+		return fields;
+	}
+
+	const value = makeFromHeaderStructure(fields);
 
 	return combineUint8Arrays([
-		matroskaToHex(getIdForName(struct.name)),
-		getVariableInt(value.length),
+		matroskaToHex(getIdForName(fields.type)),
+		getVariableInt(value.length, fields.minVintWidth),
 		value,
 	]);
 };
 
-const combineUint8Arrays = (arrays: Uint8Array[]) => {
+export const combineUint8Arrays = (arrays: Uint8Array[]) => {
+	if (arrays.length === 0) {
+		return new Uint8Array([]);
+	}
+
+	if (arrays.length === 1) {
+		return arrays[0];
+	}
+
 	let totalLength = 0;
 	for (const array of arrays) {
 		totalLength += array.length;

package/src/boxes/webm/parse-ebml.ts

@@ -1,12 +1,20 @@
-import type {BufferIterator} from '../../buffer-iterator';
+import {registerTrack} from '../../add-new-matroska-tracks';
+import {type BufferIterator} from '../../buffer-iterator';
+import type {ParserContext} from '../../parser-context';
+import type {VideoSample} from '../../webcodec-sample-types';
+import {getSampleFromBlock} from './get-sample-from-block';
+import {getTrack} from './get-track';
 import type {PossibleEbml} from './segments/all-segments';
-import {ebmlMap, type Ebml, type EbmlParsed} from './segments/all-segments';
+import {ebmlMap} from './segments/all-segments';
 
-type Prettify<T> = {
+export type Prettify<T> = {
 	[K in keyof T]: T[K];
 } & {};
 
-export const parseEbml = (iterator: BufferIterator): Prettify<PossibleEbml> => {
+export const parseEbml = async (
+	iterator: BufferIterator,
+	parserContext: ParserContext,
+): Promise<Prettify<PossibleEbml>> => {
 	const hex = iterator.getMatroskaSegmentId();
 	if (hex === null) {
 		throw new Error(
@@ -21,7 +29,10 @@ export const parseEbml = (iterator: BufferIterator): Prettify<PossibleEbml> => {
 		);
 	}
 
+	const off = iterator.counter.getOffset();
 	const size = iterator.getVint();
+	const minVintWidth = iterator.counter.getOffset() - off;
+
 	if (size === null) {
 		throw new Error(
 			'Not enough bytes left to parse EBML - this should not happen',
@@ -29,9 +40,17 @@ export const parseEbml = (iterator: BufferIterator): Prettify<PossibleEbml> => {
 	}
 
 	if (hasInMap.type === 'uint') {
+		const beforeUintOffset = iterator.counter.getOffset();
 		const value = iterator.getUint(size);
 
-		return {type: hasInMap.name, value, hex};
+		return {
+			type: hasInMap.name,
+			value: {
+				value,
+				byteLength: iterator.counter.getOffset() - beforeUintOffset,
+			},
+			minVintWidth,
+		};
 	}
 
 	if (hasInMap.type === 'string') {
@@ -40,7 +59,7 @@ export const parseEbml = (iterator: BufferIterator): Prettify<PossibleEbml> => {
 		return {
 			type: hasInMap.name,
 			value,
-			hex,
+			minVintWidth,
 		};
 	}
 
@@ -49,29 +68,49 @@ export const parseEbml = (iterator: BufferIterator): Prettify<PossibleEbml> => {
 
 		return {
 			type: hasInMap.name,
-			value,
-			hex,
+			value: {
+				value,
+				size: size === 4 ? '32' : '64',
+			},
+			minVintWidth,
 		};
 	}
 
-	if (hasInMap.type === 'void') {
-		iterator.discard(size);
+	if (hasInMap.type === 'hex-string') {
+		return {
+			type: hasInMap.name,
+			value:
+				'0x' +
+				[...iterator.getSlice(size)]
+					.map((b) => b.toString(16).padStart(2, '0'))
+					.join(''),
+			minVintWidth,
+		};
+	}
 
+	if (hasInMap.type === 'uint8array') {
 		return {
 			type: hasInMap.name,
-			value: undefined,
-			hex,
+			value: iterator.getSlice(size),
+			minVintWidth,
 		};
 	}
 
 	if (hasInMap.type === 'children') {
-		const children: EbmlParsed<Ebml>[] = [];
+		const children: PossibleEbml[] = [];
 		const startOffset = iterator.counter.getOffset();
 
 		// eslint-disable-next-line no-constant-condition
 		while (true) {
-			const value = parseEbml(iterator);
-			children.push(value);
+			const offset = iterator.counter.getOffset();
+			const value = await parseEbml(iterator, parserContext);
+			const remapped = await postprocessEbml({
+				offset,
+				ebml: value,
+				parserContext,
+			});
+			children.push(remapped);
+
 			const offsetNow = iterator.counter.getOffset();
 
 			if (offsetNow - startOffset > size) {
@@ -85,9 +124,124 @@ export const parseEbml = (iterator: BufferIterator): Prettify<PossibleEbml> => {
 			}
 		}
 
-		return {type: hasInMap.name, value: children as EbmlParsed<Ebml>[], hex};
+		return {type: hasInMap.name, value: children, minVintWidth};
 	}
 
 	// @ts-expect-error
 	throw new Error(`Unknown segment type ${hasInMap.type}`);
 };
+
+export const postprocessEbml = async ({
+	offset,
+	ebml,
+	parserContext,
+}: {
+	offset: number;
+	ebml: Prettify<PossibleEbml>;
+	parserContext: ParserContext;
+}): Promise<Prettify<PossibleEbml>> => {
+	if (ebml.type === 'TimestampScale') {
+		parserContext.parserState.setTimescale(ebml.value.value);
+	}
+
+	if (ebml.type === 'TrackEntry') {
+		parserContext.parserState.onTrackEntrySegment(ebml);
+
+		const track = getTrack({
+			track: ebml,
+			timescale: parserContext.parserState.getTimescale(),
+		});
+
+		if (track) {
+			await registerTrack({
+				state: parserContext.parserState,
+				options: parserContext,
+				track,
+			});
+		}
+	}
+
+	if (ebml.type === 'Timestamp') {
+		parserContext.parserState.setTimestampOffset(offset, ebml.value.value);
+	}
+
+	if (ebml.type === 'Block' || ebml.type === 'SimpleBlock') {
+		const sample = getSampleFromBlock(ebml, parserContext, offset);
+
+		if (sample.type === 'video-sample' && parserContext.nullifySamples) {
+			await parserContext.parserState.onVideoSample(
+				sample.videoSample.trackId,
+				sample.videoSample,
+			);
+			return {
+				type: 'Block',
+				value: new Uint8Array([]),
+				minVintWidth: ebml.minVintWidth,
+			};
+		}
+
+		if (sample.type === 'audio-sample' && parserContext.nullifySamples) {
+			await parserContext.parserState.onAudioSample(
+				sample.audioSample.trackId,
+				sample.audioSample,
+			);
+			return {
+				type: 'Block',
+				value: new Uint8Array([]),
+				minVintWidth: ebml.minVintWidth,
+			};
+		}
+
+		if (sample.type === 'no-sample' && parserContext.nullifySamples) {
+			return {
+				type: 'Block',
+				value: new Uint8Array([]),
+				minVintWidth: ebml.minVintWidth,
+			};
+		}
+	}
+
+	if (ebml.type === 'BlockGroup') {
+		// Blocks don't have information about keyframes.
+		// https://ffmpeg.org/pipermail/ffmpeg-devel/2015-June/173825.html
+		// "For Blocks, keyframes is
+		// inferred by the absence of ReferenceBlock element (as done by matroskadec).""
+
+		const block = ebml.value.find(
+			(c) => c.type === 'SimpleBlock' || c.type === 'Block',
+		);
+		if (!block || (block.type !== 'SimpleBlock' && block.type !== 'Block')) {
+			throw new Error('Expected block segment');
+		}
+
+		const hasReferenceBlock = ebml.value.find(
+			(c) => c.type === 'ReferenceBlock',
+		);
+
+		const sample =
+			block.value.length === 0
+				? null
+				: getSampleFromBlock(block, parserContext, offset);
+
+		if (sample && sample.type === 'partial-video-sample') {
+			const completeFrame: VideoSample = {
+				...sample.partialVideoSample,
+				type: hasReferenceBlock ? 'delta' : 'key',
+			};
+			await parserContext.parserState.onVideoSample(
+				sample.partialVideoSample.trackId,
+				completeFrame,
+			);
+		}
+
+		if (parserContext.nullifySamples) {
+			return {
+				type: 'BlockGroup',
+				value: [],
+				minVintWidth: ebml.minVintWidth,
+			};
+		}
+	}
+
+	return ebml;
+};