@remotion/media-parser 4.0.199 → 4.0.201

package/src/boxes/webm/segments/track-entry.ts

@@ -1,6 +1,9 @@
 import type {BufferIterator} from '../../../buffer-iterator';
 import type {ParserContext} from '../../../parser-context';
+import type {VideoSample} from '../../../webcodec-sample-types';
 import type {MatroskaSegment} from '../segments';
+import type {matroskaElements} from './all-segments';
+import {parseBlockFlags} from './block-simple-block-flags';
 import {expectChildren} from './parse-children';
 
 export type TrackEntrySegment = {
@@ -106,6 +109,11 @@ export type CodecSegment = {
 	codec: string;
 };
 
+export type TrackInfo = {
+	codec: string;
+	trackTimescale: number | null;
+};
+
 export const parseCodecSegment = (
 	iterator: BufferIterator,
 	length: number,
@@ -192,7 +200,7 @@ export const parseDefaultDurationSegment = (
 	iterator: BufferIterator,
 	length: number,
 ): DefaultDurationSegment => {
-	const defaultDuration = iterator.getDecimalBytes(length);
+	const defaultDuration = iterator.getUint(length);
 
 	return {
 		type: 'default-duration-segment',
@@ -566,80 +574,110 @@ export const parseTimestampSegment = (
 	iterator: BufferIterator,
 	length: number,
 ): TimestampSegment => {
-	if (length > 2) {
-		throw new Error('Expected timestamp segment to be 1 byte or 2 bytes');
-	}
-
-	const value = length === 2 ? iterator.getUint16() : iterator.getUint8();
-
 	return {
 		type: 'timestamp-segment',
-		timestamp: value,
+		timestamp: iterator.getUint(length),
 	};
 };
 
-export type SimpleBlockSegment = {
-	type: 'simple-block-segment';
+export type SimpleBlockOrBlockSegment = {
+	type: 'simple-block-or-block-segment';
 	length: number;
 	trackNumber: number;
-	timecode: number;
-	headerFlags: number;
-	keyframe: boolean;
-	lacing: [number, number];
+	timecodeInMicroseconds: number;
+	keyframe: boolean | null;
+	lacing: number;
 	invisible: boolean;
-	children: MatroskaSegment[];
+	videoSample: Omit<VideoSample, 'type'> | null;
 };
 
 export type GetTracks = () => TrackEntrySegment[];
 
-export const parseSimpleBlockSegment = async ({
+export const parseSimpleBlockOrBlockSegment = async ({
 	iterator,
 	length,
 	parserContext,
+	type,
 }: {
 	iterator: BufferIterator;
 	length: number;
 	parserContext: ParserContext;
-}): Promise<SimpleBlockSegment> => {
+	type:
+		| (typeof matroskaElements)['Block']
+		| (typeof matroskaElements)['SimpleBlock'];
+}): Promise<SimpleBlockOrBlockSegment> => {
 	const start = iterator.counter.getOffset();
 	const trackNumber = iterator.getVint();
-	const timecode = iterator.getUint16();
-	const headerFlags = iterator.getUint8();
+	if (trackNumber === null) {
+		throw new Error('Not enough data to get track number, should not happen');
+	}
+
+	const timecodeRelativeToCluster = iterator.getUint16();
+
+	const {invisible, lacing, keyframe} = parseBlockFlags(iterator, type);
+
+	const {codec, trackTimescale} =
+		parserContext.parserState.getTrackInfoByNumber(trackNumber);
 
-	const invisible = Boolean((headerFlags >> 5) & 1);
-	const pos6 = (headerFlags >> 6) & 1;
-	const pos7 = (headerFlags >> 6) & 1;
-	const keyframe = Boolean((headerFlags >> 7) & 1);
+	const clusterOffset =
+		parserContext.parserState.getTimestampOffsetForByteOffset(
+			iterator.counter.getOffset(),
+		);
+
+	const timescale = parserContext.parserState.getTimescale();
+
+	if (clusterOffset === undefined) {
+		throw new Error(
+			'Could not find offset for byte offset ' + iterator.counter.getOffset(),
+		);
+	}
 
-	const codec = parserContext.parserState.getTrackInfoByNumber(trackNumber);
+	// https://github.com/hubblec4/Matroska-Chapters-Specs/blob/master/notes.md/#timestampscale
+	// The TimestampScale Element is used to calculate the Raw Timestamp of a Block. The timestamp is obtained by adding the Block's timestamp to the Cluster's Timestamp Element, and then multiplying that result by the TimestampScale. The result will be the Block's Raw Timestamp in nanoseconds.
+	const timecodeInNanoSeconds =
+		(timecodeRelativeToCluster + clusterOffset) *
+		timescale *
+		(trackTimescale ?? 1);
+
+	// Timecode should be in microseconds
+	const timecodeInMicroseconds = timecodeInNanoSeconds / 1000;
 
 	if (!codec) {
 		throw new Error('Could not find codec for track ' + trackNumber);
 	}
 
-	const children: MatroskaSegment[] = [];
+	const remainingNow = length - (iterator.counter.getOffset() - start);
 
-	if (codec.codec.startsWith('V_')) {
-		const remainingNow = length - (iterator.counter.getOffset() - start);
+	let videoSample: Omit<VideoSample, 'type'> | null = null;
 
-		await parserContext.parserState.onVideoSample(trackNumber, {
+	if (codec.startsWith('V_')) {
+		const partialVideoSample: Omit<VideoSample, 'type'> = {
 			data: iterator.getSlice(remainingNow),
 			cts: null,
 			dts: null,
 			duration: undefined,
-			type: keyframe ? 'key' : 'delta',
 			trackId: trackNumber,
-			timestamp: timecode,
-		});
+			timestamp: timecodeInMicroseconds,
+		};
+
+		if (keyframe === null) {
+			// If we don't know if this is a keyframe, we know after we emit the BlockGroup
+			videoSample = partialVideoSample;
+		} else {
+			const sample: VideoSample = {
+				...partialVideoSample,
+				type: keyframe ? 'key' : 'delta',
+			};
+
+			await parserContext.parserState.onVideoSample(trackNumber, sample);
+		}
 	}
 
-	if (codec.codec.startsWith('A_')) {
-		const vorbisRemaining = length - (iterator.counter.getOffset() - start);
+	if (codec.startsWith('A_')) {
 		await parserContext.parserState.onAudioSample(trackNumber, {
-			data: iterator.getSlice(vorbisRemaining),
-			offset: timecode,
+			data: iterator.getSlice(remainingNow),
 			trackId: trackNumber,
-			timestamp: timecode,
+			timestamp: timecodeInMicroseconds,
 			type: 'key',
 		});
 	}
@@ -650,15 +688,14 @@ export const parseSimpleBlockSegment = async ({
 	}
 
 	return {
-		type: 'simple-block-segment',
+		type: 'simple-block-or-block-segment',
 		length,
 		trackNumber,
-		timecode,
-		headerFlags,
+		timecodeInMicroseconds,
 		keyframe,
-		lacing: [pos6, pos7],
+		lacing,
 		invisible,
-		children,
+		videoSample,
 	};
 };
 
@@ -705,6 +742,53 @@ export const parseBlockGroupSegment = async (
 	};
 };
 
+export type ReferenceBlockSegment = {
+	type: 'reference-block-segment';
+	referenceBlock: number;
+};
+
+export const parseReferenceBlockSegment = (
+	iterator: BufferIterator,
+	length: number,
+): ReferenceBlockSegment => {
+	if (length > 4) {
+		throw new Error(
+			`Expected reference block segment to be 4 bytes, but got ${length}`,
+		);
+	}
+
+	const referenceBlock =
+		length === 4
+			? iterator.getUint32()
+			: length === 3
+				? iterator.getUint24()
+				: length === 2
+					? iterator.getUint16()
+					: iterator.getUint8();
+
+	return {
+		type: 'reference-block-segment',
+		referenceBlock,
+	};
+};
+
+export type BlockAdditionsSegment = {
+	type: 'block-additions-segment';
+	blockAdditions: Uint8Array;
+};
+
+export const parseBlockAdditionsSegment = (
+	iterator: BufferIterator,
+	length: number,
+): BlockAdditionsSegment => {
+	const blockAdditions = iterator.getSlice(length);
+
+	return {
+		type: 'block-additions-segment',
+		blockAdditions,
+	};
+};
+
 export type BlockElement = {
 	type: 'block-element-segment';
 	length: number;

package/src/boxes/webm/segments.ts

@@ -2,7 +2,10 @@ import {registerTrack} from '../../add-new-matroska-tracks';
 import type {BufferIterator} from '../../buffer-iterator';
 import type {ParseResult} from '../../parse-result';
 import type {ParserContext} from '../../parser-context';
+import type {VideoSample} from '../../webcodec-sample-types';
 import {getTrack} from './get-track';
+import {parseEbml} from './parse-ebml';
+import type {PossibleEbml} from './segments/all-segments';
 import {matroskaElements} from './segments/all-segments';
 import type {DurationSegment} from './segments/duration';
 import {parseDurationSegment} from './segments/duration';
@@ -29,6 +32,7 @@ import type {
 	AlphaModeSegment,
 	AudioSegment,
 	BitDepthSegment,
+	BlockAdditionsSegment,
 	BlockElement,
 	BlockGroupSegment,
 	ChannelsSegment,
@@ -46,9 +50,10 @@ import type {
 	InterlacedSegment,
 	LanguageSegment,
 	MaxBlockAdditionId,
+	ReferenceBlockSegment,
 	SamplingFrequencySegment,
 	SegmentUUIDSegment,
-	SimpleBlockSegment,
+	SimpleBlockOrBlockSegment,
 	TagSegment,
 	TagsSegment,
 	TimestampSegment,
@@ -64,6 +69,7 @@ import {
 	parseAlphaModeSegment,
 	parseAudioSegment,
 	parseBitDepthSegment,
+	parseBlockAdditionsSegment,
 	parseBlockElementSegment,
 	parseBlockGroupSegment,
 	parseChannelsSegment,
@@ -80,9 +86,10 @@ import {
 	parseInterlacedSegment,
 	parseLanguageSegment,
 	parseMaxBlockAdditionId,
+	parseReferenceBlockSegment,
 	parseSamplingFrequencySegment,
 	parseSegmentUUIDSegment,
-	parseSimpleBlockSegment,
+	parseSimpleBlockOrBlockSegment,
 	parseTagSegment,
 	parseTagsSegment,
 	parseTimestampSegment,
@@ -96,8 +103,6 @@ import {
 } from './segments/track-entry';
 import type {TracksSegment} from './segments/tracks';
 import {parseTracksSegment} from './segments/tracks';
-import type {UnknownSegment} from './segments/unknown';
-import {parseUnknownSegment} from './segments/unknown';
 import type {VoidSegment} from './segments/void';
 import {parseVoidSegment} from './segments/void';
 import type {WritingAppSegment} from './segments/writing';
@@ -105,7 +110,6 @@ import {parseWritingSegment} from './segments/writing';
 
 export type MatroskaSegment =
 	| MainSegment
-	| UnknownSegment
 	| SeekHeadSegment
 	| SeekSegment
 	| SeekPositionSegment
@@ -142,14 +146,17 @@ export type MatroskaSegment =
 	| TagSegment
 	| ClusterSegment
 	| TimestampSegment
-	| SimpleBlockSegment
+	| SimpleBlockOrBlockSegment
 	| BlockGroupSegment
 	| BlockElement
 	| SeekIdSegment
 	| AudioSegment
 	| SamplingFrequencySegment
 	| ChannelsSegment
-	| BitDepthSegment;
+	| BitDepthSegment
+	| ReferenceBlockSegment
+	| BlockAdditionsSegment
+	| PossibleEbml;
 
 export type OnTrackEntrySegment = (trackEntry: TrackEntrySegment) => void;
 
@@ -158,39 +165,24 @@ const parseSegment = async ({
 	iterator,
 	length,
 	parserContext,
+	headerReadSoFar,
 }: {
 	segmentId: string;
 	iterator: BufferIterator;
 	length: number;
 	parserContext: ParserContext;
+	headerReadSoFar: number;
 }): Promise<Promise<MatroskaSegment> | MatroskaSegment> => {
 	if (length === 0) {
 		throw new Error(`Expected length of ${segmentId} to be greater than 0`);
 	}
 
-	if (segmentId === '0x') {
-		return {
-			type: 'unknown-segment',
-			id: segmentId,
-		};
-	}
-
-	// Log this to debug
-	/*
-	console.log(
-		'segmentId',
-		segmentId,
-		getSegmentName(segmentId),
-		iterator.counter.getOffset(),
-	);
-	*/
-
 	if (segmentId === '0x114d9b74') {
 		return parseSeekHeadSegment(iterator, length, parserContext);
 	}
 
 	if (segmentId === '0x53ab') {
-		return parseSeekIdSegment(iterator);
+		return parseSeekIdSegment(iterator, length);
 	}
 
 	if (segmentId === '0x4dbb') {
@@ -360,42 +352,92 @@ const parseSegment = async ({
 		return parseBitDepthSegment(iterator, length);
 	}
 
-	if (segmentId === '0xe7') {
-		return parseTimestampSegment(iterator, length);
+	if (segmentId === matroskaElements.Timestamp) {
+		const offset = iterator.counter.getOffset();
+		const timestampSegment = parseTimestampSegment(iterator, length);
+
+		parserContext.parserState.setTimestampOffset(
+			offset,
+			timestampSegment.timestamp,
+		);
+
+		return timestampSegment;
 	}
 
-	if (segmentId === '0xa3') {
-		return parseSimpleBlockSegment({
+	if (
+		segmentId === matroskaElements.SimpleBlock ||
+		segmentId === matroskaElements.Block
+	) {
+		return parseSimpleBlockOrBlockSegment({
 			iterator,
 			length,
 			parserContext,
+			type: segmentId,
 		});
 	}
 
+	if (segmentId === matroskaElements.ReferenceBlock) {
+		return parseReferenceBlockSegment(iterator, length);
+	}
+
+	if (segmentId === matroskaElements.BlockAdditions) {
+		return parseBlockAdditionsSegment(iterator, length);
+	}
+
 	if (segmentId === '0xa0') {
-		return parseBlockGroupSegment(iterator, length, parserContext);
+		const blockGroup = await parseBlockGroupSegment(
+			iterator,
+			length,
+			parserContext,
+		);
+
+		// Blocks don't have information about keyframes.
+		// https://ffmpeg.org/pipermail/ffmpeg-devel/2015-June/173825.html
+		// "For Blocks, keyframes is
+		// inferred by the absence of ReferenceBlock element (as done by matroskadec).""
+
+		const block = blockGroup.children.find(
+			(c) => c.type === 'simple-block-or-block-segment',
+		);
+		if (!block || block.type !== 'simple-block-or-block-segment') {
+			throw new Error('Expected block segment');
+		}
+
+		const hasReferenceBlock = blockGroup.children.find(
+			(c) => c.type === 'reference-block-segment',
+		);
+
+		const partialVideoSample = block.videoSample;
+
+		if (partialVideoSample) {
+			const completeFrame: VideoSample = {
+				...partialVideoSample,
+				type: hasReferenceBlock ? 'delta' : 'key',
+			};
+			await parserContext.parserState.onVideoSample(
+				partialVideoSample.trackId,
+				completeFrame,
+			);
+		}
+
+		return blockGroup;
 	}
 
 	if (segmentId === '0xa1') {
 		return parseBlockElementSegment(iterator, length);
 	}
 
-	const bytesRemaining = iterator.byteLength() - iterator.counter.getOffset();
-	const toDiscard = Math.min(
-		bytesRemaining,
-		length > 0 ? length : bytesRemaining,
-	);
+	iterator.counter.decrement(headerReadSoFar);
 
-	const child = parseUnknownSegment(iterator, segmentId, toDiscard);
-	return child;
+	return parseEbml(iterator);
 };
 
 export const expectSegment = async (
 	iterator: BufferIterator,
 	parserContext: ParserContext,
 ): Promise<ParseResult> => {
-	const bytesRemaining_ = iterator.bytesRemaining();
-	if (bytesRemaining_ === 0) {
+	const offset = iterator.counter.getOffset();
+	if (iterator.bytesRemaining() === 0) {
 		return {
 			status: 'incomplete',
 			segments: [],
@@ -406,9 +448,33 @@ export const expectSegment = async (
 		};
 	}
 
-	const offset = iterator.counter.getOffset();
 	const segmentId = iterator.getMatroskaSegmentId();
+
+	if (segmentId === null) {
+		iterator.counter.decrement(iterator.counter.getOffset() - offset);
+		return {
+			status: 'incomplete',
+			segments: [],
+			continueParsing: () => {
+				return Promise.resolve(expectSegment(iterator, parserContext));
+			},
+			skipTo: null,
+		};
+	}
+
 	const length = iterator.getVint();
+	if (length === null) {
+		iterator.counter.decrement(iterator.counter.getOffset() - offset);
+		return {
+			status: 'incomplete',
+			segments: [],
+			continueParsing: () => {
+				return Promise.resolve(expectSegment(iterator, parserContext));
+			},
+			skipTo: null,
+		};
+	}
+
 	const bytesRemainingNow =
 		iterator.byteLength() - iterator.counter.getOffset();
 
@@ -463,6 +529,7 @@ export const expectSegment = async (
 		iterator,
 		length,
 		parserContext,
+		headerReadSoFar: iterator.counter.getOffset() - offset,
 	});
 
 	return {

package/src/boxes/webm/traversal.ts

@@ -24,6 +24,19 @@ export const getTrackCodec = (track: TrackEntrySegment) => {
 	return child ?? null;
 };
 
+export const getTrackTimestampScale = (track: TrackEntrySegment) => {
+	const child = track.children.find((b) => b.type === 'TrackTimestampScale');
+	if (!child) {
+		return null;
+	}
+
+	if (child.type !== 'TrackTimestampScale') {
+		throw new Error('Expected TrackTimestampScale');
+	}
+
+	return child.value;
+};
+
 export const getTrackByNumber = (tracks: TrackEntrySegment[], id: number) => {
 	return tracks.find((track) => {
 		const trackNumber = getTrackNumber(track);