@remotion/media-parser 4.0.200 → 4.0.201

package/src/boxes/webm/parse-webm-header.ts

@@ -1,22 +1,18 @@
 import type {BufferIterator} from '../../buffer-iterator';
 import type {ParseResult} from '../../parse-result';
 import type {ParserContext} from '../../parser-context';
-import {expectSegment} from './segments';
+import {expectChildren} from './segments/parse-children';
 
 // Parsing according to https://darkcoding.net/software/reading-mediarecorders-webm-opus-output/
 export const parseWebm = (
 	counter: BufferIterator,
 	parserContext: ParserContext,
 ): Promise<ParseResult> => {
-	counter.discard(4);
-	const length = counter.getVint();
-
-	if (length !== 31 && length !== 35) {
-		throw new Error(`Expected header length 31 or 25, got ${length}`);
-	}
-
-	// Discard header for now
-	counter.discard(length);
-
-	return expectSegment(counter, parserContext);
+	return expectChildren({
+		iterator: counter,
+		length: Infinity,
+		initialChildren: [],
+		wrap: null,
+		parserContext,
+	});
 };

package/src/boxes/webm/segments/all-segments.ts

@@ -1,7 +1,8 @@
 export const matroskaElements = {
+	Header: '0x1a45dfa3',
 	EBMLMaxIDLength: '0x42f2',
 	EBMLVersion: '0x4286',
-	EBMLReadVersion: '0x42F7',
+	EBMLReadVersion: '0x42f7',
 	EBMLMaxSizeLength: '0x42f3',
 	DocType: '0x4282',
 	DocTypeVersion: '0x4287',
@@ -288,3 +289,223 @@ export const getSegmentName = (id: string) => {
 		([, value]) => value === id,
 	)?.[0];
 };
+
+export const getIdForName = (name: string) => {
+	const value = Object.entries(matroskaElements).find(
+		([key]) => key === name,
+	)?.[1];
+	if (!value) {
+		throw new Error(`Could not find id for name ${name}`);
+	}
+
+	return value;
+};
+
+export type MatroskaKey = keyof typeof matroskaElements;
+
+export type MatroskaElement = (typeof matroskaElements)[MatroskaKey];
+
+type EbmlType = 'string';
+
+export type EbmlWithChildren = {
+	name: MatroskaKey;
+	type: 'children';
+	children: HeaderStructure;
+};
+
+export type EbmlWithUint8 = {
+	name: MatroskaKey;
+	type: 'uint';
+};
+
+export type EbmlWithHexString = {
+	name: MatroskaKey;
+	type: 'hex-string';
+};
+
+export type EbmlWithString = {
+	name: MatroskaKey;
+	type: EbmlType;
+};
+
+export type EbmlWithVoid = {
+	name: MatroskaKey;
+	type: 'void';
+};
+
+export type EbmlWithFloat = {
+	name: MatroskaKey;
+	type: 'float';
+};
+
+export type Ebml =
+	| EbmlWithString
+	| EbmlWithUint8
+	| EbmlWithChildren
+	| EbmlWithVoid
+	| EbmlWithFloat
+	| EbmlWithHexString;
+
+export const ebmlVersion = {
+	name: 'EBMLVersion',
+	type: 'uint',
+} satisfies Ebml;
+
+export const ebmlReadVersion = {
+	name: 'EBMLReadVersion',
+	type: 'uint',
+} satisfies Ebml;
+
+export const ebmlMaxIdLength = {
+	name: 'EBMLMaxIDLength',
+	type: 'uint',
+} satisfies Ebml;
+
+export const ebmlMaxSizeLength = {
+	name: 'EBMLMaxSizeLength',
+	type: 'uint',
+} satisfies Ebml;
+
+export const docType = {
+	name: 'DocType',
+	type: 'string',
+} satisfies Ebml;
+
+export const docTypeVersion = {
+	name: 'DocTypeVersion',
+	type: 'uint',
+} satisfies Ebml;
+
+export const docTypeReadVersion = {
+	name: 'DocTypeReadVersion',
+	type: 'uint',
+} satisfies Ebml;
+
+export const voidEbml = {
+	name: 'Void',
+	type: 'void',
+} satisfies Ebml;
+
+export type EmblTypes = {
+	uint: number;
+	float: number;
+	string: string;
+	children: HeaderStructure;
+	void: undefined;
+	'hex-string': string;
+};
+
+export type HeaderStructure = Ebml[];
+
+export const matroskaHeaderStructure = [
+	ebmlVersion,
+	ebmlReadVersion,
+	ebmlMaxIdLength,
+	ebmlMaxSizeLength,
+	docType,
+	docTypeVersion,
+	docTypeReadVersion,
+] as const satisfies HeaderStructure;
+
+export const matroskaHeader = {
+	name: 'Header',
+	type: 'children',
+	children: matroskaHeaderStructure,
+} as const satisfies Ebml;
+
+export const seekId = {
+	name: 'SeekID',
+	type: 'hex-string',
+} as const satisfies Ebml;
+
+export const seekPosition = {
+	name: 'SeekPosition',
+	type: 'uint',
+} as const satisfies Ebml;
+
+export const seek = {
+	name: 'Seek',
+	type: 'children',
+	children: [seekId, seekPosition],
+} as const satisfies Ebml;
+
+export const voidHeader = {
+	name: 'Void',
+	type: 'void',
+} as const satisfies Ebml;
+
+export type EbmlValue<T extends Ebml> = T extends EbmlWithUint8
+	? number
+	: T extends EbmlWithVoid
+		? undefined
+		: T extends EbmlWithString
+			? string
+			: T extends EbmlWithFloat
+				? number
+				: EbmlParsed<Ebml>[];
+
+export type EbmlParsed<T extends Ebml> = {
+	type: T['name'];
+	value: EbmlValue<T>;
+	hex: string;
+};
+
+export const ebmlMap = {
+	[matroskaElements.Header]: matroskaHeader,
+	[matroskaElements.DocType]: docType,
+	[matroskaElements.DocTypeVersion]: docTypeVersion,
+	[matroskaElements.DocTypeReadVersion]: docTypeReadVersion,
+	[matroskaElements.EBMLVersion]: ebmlVersion,
+	[matroskaElements.EBMLReadVersion]: ebmlReadVersion,
+	[matroskaElements.EBMLMaxIDLength]: ebmlMaxIdLength,
+	[matroskaElements.EBMLMaxSizeLength]: ebmlMaxSizeLength,
+	[matroskaElements.Void]: voidEbml,
+	[matroskaElements.Cues]: {
+		name: 'Cues',
+		type: 'void',
+	},
+	[matroskaElements.DateUTC]: {
+		name: 'DateUTC',
+		type: 'void',
+	},
+	[matroskaElements.TrackTimestampScale]: {
+		name: 'TrackTimestampScale',
+		type: 'float',
+	},
+	[matroskaElements.CodecDelay]: {
+		name: 'CodecDelay',
+		type: 'void',
+	},
+	[matroskaElements.SeekPreRoll]: {
+		name: 'SeekPreRoll',
+		type: 'void',
+	},
+	[matroskaElements.DiscardPadding]: {
+		name: 'DiscardPadding',
+		type: 'void',
+	},
+	[matroskaElements.OutputSamplingFrequency]: {
+		name: 'OutputSamplingFrequency',
+		type: 'void',
+	},
+	[matroskaElements.CodecName]: {
+		name: 'CodecName',
+		type: 'void',
+	},
+	[matroskaElements.Position]: {
+		name: 'Position',
+		type: 'void',
+	},
+	[matroskaElements.SliceDuration]: {
+		name: 'SliceDuration',
+		type: 'void',
+	},
+} as const satisfies Partial<Record<MatroskaElement, Ebml>>;
+
+export type PossibleEbml = {
+	[key in keyof typeof ebmlMap]: {
+		type: (typeof ebmlMap)[key]['name'];
+		value: EbmlValue<(typeof ebmlMap)[key]>;
+		hex: string;
+	};
+}[keyof typeof ebmlMap];

package/src/boxes/webm/segments/seek-position.ts

@@ -9,7 +9,7 @@ export const parseSeekPositionSegment = (
 	iterator: BufferIterator,
 	length: number,
 ): SeekPositionSegment => {
-	const seekPosition = iterator.getDecimalBytes(length);
+	const seekPosition = iterator.getUint(length);
 
 	return {
 		type: 'seek-position-segment',

package/src/boxes/webm/segments/seek.ts

@@ -35,8 +35,18 @@ export type SeekIdSegment = {
 	seekId: string;
 };
 
-export const parseSeekIdSegment = (iterator: BufferIterator): SeekIdSegment => {
-	const seekId = iterator.getMatroskaSegmentId();
+export const parseSeekIdSegment = (
+	iterator: BufferIterator,
+	length: number,
+): SeekIdSegment => {
+	const seekId =
+		'0x' +
+		[...iterator.getSlice(length)]
+			.map((b) => b.toString(16).padStart(2, '0'))
+			.join('');
+	if (seekId === null) {
+		throw new Error('Not enough bytes to parse seek id');
+	}
 
 	return {
 		type: 'seek-id-segment',

package/src/boxes/webm/segments/timestamp-scale.ts

@@ -8,7 +8,7 @@ export type TimestampScaleSegment = {
 export const parseTimestampScaleSegment = (
 	iterator: BufferIterator,
 ): TimestampScaleSegment => {
-	const timestampScale = iterator.getDecimalBytes(3);
+	const timestampScale = iterator.getUint(3);
 
 	return {
 		type: 'timestamp-scale-segment',

package/src/boxes/webm/segments/track-entry.ts

@@ -109,6 +109,11 @@ export type CodecSegment = {
 	codec: string;
 };
 
+export type TrackInfo = {
+	codec: string;
+	trackTimescale: number | null;
+};
+
 export const parseCodecSegment = (
 	iterator: BufferIterator,
 	length: number,
@@ -195,7 +200,7 @@ export const parseDefaultDurationSegment = (
 	iterator: BufferIterator,
 	length: number,
 ): DefaultDurationSegment => {
-	const defaultDuration = iterator.getDecimalBytes(length);
+	const defaultDuration = iterator.getUint(length);
 
 	return {
 		type: 'default-duration-segment',
@@ -569,25 +574,9 @@ export const parseTimestampSegment = (
 	iterator: BufferIterator,
 	length: number,
 ): TimestampSegment => {
-	if (length > 3) {
-		throw new Error(
-			'Expected timestamp segment to be 1 byte or 2 bytes, but is ' + length,
-		);
-	}
-
-	if (length === 3) {
-		const val = iterator.getUint24();
-		return {
-			type: 'timestamp-segment',
-			timestamp: val,
-		};
-	}
-
-	const value = length === 2 ? iterator.getUint16() : iterator.getUint8();
-
 	return {
 		type: 'timestamp-segment',
-		timestamp: value,
+		timestamp: iterator.getUint(length),
 	};
 };
 
@@ -595,7 +584,7 @@ export type SimpleBlockOrBlockSegment = {
 	type: 'simple-block-or-block-segment';
 	length: number;
 	trackNumber: number;
-	timecode: number;
+	timecodeInMicroseconds: number;
 	keyframe: boolean | null;
 	lacing: number;
 	invisible: boolean;
@@ -619,23 +608,39 @@ export const parseSimpleBlockOrBlockSegment = async ({
 }): Promise<SimpleBlockOrBlockSegment> => {
 	const start = iterator.counter.getOffset();
 	const trackNumber = iterator.getVint();
+	if (trackNumber === null) {
+		throw new Error('Not enough data to get track number, should not happen');
+	}
+
 	const timecodeRelativeToCluster = iterator.getUint16();
 
 	const {invisible, lacing, keyframe} = parseBlockFlags(iterator, type);
 
-	const codec = parserContext.parserState.getTrackInfoByNumber(trackNumber);
+	const {codec, trackTimescale} =
+		parserContext.parserState.getTrackInfoByNumber(trackNumber);
+
 	const clusterOffset =
 		parserContext.parserState.getTimestampOffsetForByteOffset(
 			iterator.counter.getOffset(),
 		);
 
+	const timescale = parserContext.parserState.getTimescale();
+
 	if (clusterOffset === undefined) {
 		throw new Error(
 			'Could not find offset for byte offset ' + iterator.counter.getOffset(),
 		);
 	}
 
-	const timecode = timecodeRelativeToCluster + clusterOffset;
+	// https://github.com/hubblec4/Matroska-Chapters-Specs/blob/master/notes.md/#timestampscale
+	// The TimestampScale Element is used to calculate the Raw Timestamp of a Block. The timestamp is obtained by adding the Block's timestamp to the Cluster's Timestamp Element, and then multiplying that result by the TimestampScale. The result will be the Block's Raw Timestamp in nanoseconds.
+	const timecodeInNanoSeconds =
+		(timecodeRelativeToCluster + clusterOffset) *
+		timescale *
+		(trackTimescale ?? 1);
+
+	// Timecode should be in microseconds
+	const timecodeInMicroseconds = timecodeInNanoSeconds / 1000;
 
 	if (!codec) {
 		throw new Error('Could not find codec for track ' + trackNumber);
@@ -645,14 +650,14 @@ export const parseSimpleBlockOrBlockSegment = async ({
 
 	let videoSample: Omit<VideoSample, 'type'> | null = null;
 
-	if (codec.codec.startsWith('V_')) {
+	if (codec.startsWith('V_')) {
 		const partialVideoSample: Omit<VideoSample, 'type'> = {
 			data: iterator.getSlice(remainingNow),
 			cts: null,
 			dts: null,
 			duration: undefined,
 			trackId: trackNumber,
-			timestamp: timecode,
+			timestamp: timecodeInMicroseconds,
 		};
 
 		if (keyframe === null) {
@@ -668,11 +673,11 @@ export const parseSimpleBlockOrBlockSegment = async ({
 		}
 	}
 
-	if (codec.codec.startsWith('A_')) {
+	if (codec.startsWith('A_')) {
 		await parserContext.parserState.onAudioSample(trackNumber, {
 			data: iterator.getSlice(remainingNow),
 			trackId: trackNumber,
-			timestamp: timecode,
+			timestamp: timecodeInMicroseconds,
 			type: 'key',
 		});
 	}
@@ -686,7 +691,7 @@ export const parseSimpleBlockOrBlockSegment = async ({
 		type: 'simple-block-or-block-segment',
 		length,
 		trackNumber,
-		timecode,
+		timecodeInMicroseconds,
 		keyframe,
 		lacing,
 		invisible,

package/src/boxes/webm/segments.ts

@@ -4,6 +4,8 @@ import type {ParseResult} from '../../parse-result';
 import type {ParserContext} from '../../parser-context';
 import type {VideoSample} from '../../webcodec-sample-types';
 import {getTrack} from './get-track';
+import {parseEbml} from './parse-ebml';
+import type {PossibleEbml} from './segments/all-segments';
 import {matroskaElements} from './segments/all-segments';
 import type {DurationSegment} from './segments/duration';
 import {parseDurationSegment} from './segments/duration';
@@ -101,8 +103,6 @@ import {
 } from './segments/track-entry';
 import type {TracksSegment} from './segments/tracks';
 import {parseTracksSegment} from './segments/tracks';
-import type {UnknownSegment} from './segments/unknown';
-import {parseUnknownSegment} from './segments/unknown';
 import type {VoidSegment} from './segments/void';
 import {parseVoidSegment} from './segments/void';
 import type {WritingAppSegment} from './segments/writing';
@@ -110,7 +110,6 @@ import {parseWritingSegment} from './segments/writing';
 
 export type MatroskaSegment =
 	| MainSegment
-	| UnknownSegment
 	| SeekHeadSegment
 	| SeekSegment
 	| SeekPositionSegment
@@ -156,7 +155,8 @@ export type MatroskaSegment =
 	| ChannelsSegment
 	| BitDepthSegment
 	| ReferenceBlockSegment
-	| BlockAdditionsSegment;
+	| BlockAdditionsSegment
+	| PossibleEbml;
 
 export type OnTrackEntrySegment = (trackEntry: TrackEntrySegment) => void;
 
@@ -165,39 +165,24 @@ const parseSegment = async ({
 	iterator,
 	length,
 	parserContext,
+	headerReadSoFar,
 }: {
 	segmentId: string;
 	iterator: BufferIterator;
 	length: number;
 	parserContext: ParserContext;
+	headerReadSoFar: number;
 }): Promise<Promise<MatroskaSegment> | MatroskaSegment> => {
 	if (length === 0) {
 		throw new Error(`Expected length of ${segmentId} to be greater than 0`);
 	}
 
-	if (segmentId === '0x') {
-		return {
-			type: 'unknown-segment',
-			id: segmentId,
-		};
-	}
-
-	// Log this to debug
-	/*
-	console.log(
-		'segmentId',
-		segmentId,
-		getSegmentName(segmentId),
-		iterator.counter.getOffset(),
-	);
-	*/
-
 	if (segmentId === '0x114d9b74') {
 		return parseSeekHeadSegment(iterator, length, parserContext);
 	}
 
 	if (segmentId === '0x53ab') {
-		return parseSeekIdSegment(iterator);
+		return parseSeekIdSegment(iterator, length);
 	}
 
 	if (segmentId === '0x4dbb') {
@@ -442,22 +427,17 @@ const parseSegment = async ({
 		return parseBlockElementSegment(iterator, length);
 	}
 
-	const bytesRemaining = iterator.byteLength() - iterator.counter.getOffset();
-	const toDiscard = Math.min(
-		bytesRemaining,
-		length > 0 ? length : bytesRemaining,
-	);
+	iterator.counter.decrement(headerReadSoFar);
 
-	const child = parseUnknownSegment(iterator, segmentId, toDiscard);
-	return child;
+	return parseEbml(iterator);
 };
 
 export const expectSegment = async (
 	iterator: BufferIterator,
 	parserContext: ParserContext,
 ): Promise<ParseResult> => {
-	const bytesRemaining_ = iterator.bytesRemaining();
-	if (bytesRemaining_ === 0) {
+	const offset = iterator.counter.getOffset();
+	if (iterator.bytesRemaining() === 0) {
 		return {
 			status: 'incomplete',
 			segments: [],
@@ -468,9 +448,33 @@ export const expectSegment = async (
 		};
 	}
 
-	const offset = iterator.counter.getOffset();
 	const segmentId = iterator.getMatroskaSegmentId();
+
+	if (segmentId === null) {
+		iterator.counter.decrement(iterator.counter.getOffset() - offset);
+		return {
+			status: 'incomplete',
+			segments: [],
+			continueParsing: () => {
+				return Promise.resolve(expectSegment(iterator, parserContext));
+			},
+			skipTo: null,
+		};
+	}
+
 	const length = iterator.getVint();
+	if (length === null) {
+		iterator.counter.decrement(iterator.counter.getOffset() - offset);
+		return {
+			status: 'incomplete',
+			segments: [],
+			continueParsing: () => {
+				return Promise.resolve(expectSegment(iterator, parserContext));
+			},
+			skipTo: null,
+		};
+	}
+
 	const bytesRemainingNow =
 		iterator.byteLength() - iterator.counter.getOffset();
 
@@ -525,6 +529,7 @@ export const expectSegment = async (
 		iterator,
 		length,
 		parserContext,
+		headerReadSoFar: iterator.counter.getOffset() - offset,
 	});
 
 	return {

package/src/boxes/webm/traversal.ts

@@ -24,6 +24,19 @@ export const getTrackCodec = (track: TrackEntrySegment) => {
 	return child ?? null;
 };
 
+export const getTrackTimestampScale = (track: TrackEntrySegment) => {
+	const child = track.children.find((b) => b.type === 'TrackTimestampScale');
+	if (!child) {
+		return null;
+	}
+
+	if (child.type !== 'TrackTimestampScale') {
+		throw new Error('Expected TrackTimestampScale');
+	}
+
+	return child.value;
+};
+
 export const getTrackByNumber = (tracks: TrackEntrySegment[], id: number) => {
 	return tracks.find((track) => {
 		const trackNumber = getTrackNumber(track);