@remotion/media-parser 4.0.199 → 4.0.200

package/dist/parser-state.js

@@ -21,7 +21,6 @@ const makeParserState = ({ hasAudioCallbacks, hasVideoCallbacks, }) => {
     };
     const videoSampleCallbacks = {};
     const audioSampleCallbacks = {};
-    let samplesThatHadToBeQueued = 0;
     const queuedAudioSamples = {};
     const queuedVideoSamples = {};
     const declinedTrackNumbers = [];
@@ -35,6 +34,25 @@ const makeParserState = ({ hasAudioCallbacks, hasVideoCallbacks, }) => {
     const setTimescale = (newTimescale) => {
         timescale = newTimescale;
     };
+    const timestampMap = new Map();
+    const setTimestampOffset = (byteOffset, timestamp) => {
+        timestampMap.set(byteOffset, timestamp);
+    };
+    const getTimestampOffsetForByteOffset = (byteOffset) => {
+        const entries = Array.from(timestampMap.entries());
+        const sortedByByteOffset = entries
+            .sort((a, b) => {
+            return a[0] - b[0];
+        })
+            .reverse();
+        for (const [offset, timestamp] of sortedByByteOffset) {
+            if (offset >= byteOffset) {
+                continue;
+            }
+            return timestamp;
+        }
+        return timestampMap.get(byteOffset);
+    };
     return {
         onTrackEntrySegment,
         getTrackInfoByNumber: (id) => trackEntries[id],
@@ -51,6 +69,8 @@ const makeParserState = ({ hasAudioCallbacks, hasVideoCallbacks, }) => {
             }
             queuedVideoSamples[id] = [];
         },
+        setTimestampOffset,
+        getTimestampOffsetForByteOffset,
         registerAudioSampleCallback: async (id, callback) => {
             var _a;
             if (callback === null) {
@@ -65,7 +85,6 @@ const makeParserState = ({ hasAudioCallbacks, hasVideoCallbacks, }) => {
             queuedAudioSamples[id] = [];
         },
         onAudioSample: async (trackId, audioSample) => {
-            var _a;
             const callback = audioSampleCallbacks[trackId];
             if (callback) {
                 await callback(audioSample);
@@ -75,15 +94,11 @@ const makeParserState = ({ hasAudioCallbacks, hasVideoCallbacks, }) => {
                     return;
                 }
                 if (!hasAudioCallbacks) {
-                    return;
+                    throw new Error('No audio callbacks registered');
                 }
-                (_a = queuedAudioSamples[trackId]) !== null && _a !== void 0 ? _a : (queuedAudioSamples[trackId] = []);
-                queuedAudioSamples[trackId].push(audioSample);
-                samplesThatHadToBeQueued++;
             }
         },
         onVideoSample: async (trackId, videoSample) => {
-            var _a;
             const callback = videoSampleCallbacks[trackId];
             if (callback) {
                 await callback(videoSample);
@@ -93,18 +108,11 @@ const makeParserState = ({ hasAudioCallbacks, hasVideoCallbacks, }) => {
                     return;
                 }
                 if (!hasVideoCallbacks) {
-                    return;
+                    throw new Error('No video callbacks registered');
                 }
-                (_a = queuedVideoSamples[trackId]) !== null && _a !== void 0 ? _a : (queuedVideoSamples[trackId] = []);
-                queuedVideoSamples[trackId].push(videoSample);
-                samplesThatHadToBeQueued++;
             }
         },
-        getInternalStats: () => {
-            return {
-                samplesThatHadToBeQueued,
-            };
-        },
+        getInternalStats: () => ({}),
         getTimescale,
         setTimescale,
     };

package/dist/webcodec-sample-types.d.ts

@@ -2,7 +2,6 @@ import type { AudioTrack, VideoTrack } from './get-tracks';
 export type AudioSample = {
     data: Uint8Array;
     timestamp: number;
-    offset: number;
     trackId: number;
     type: 'key' | 'delta';
 };

package/package.json

@@ -3,11 +3,11 @@
     "url": "https://github.com/remotion-dev/remotion/tree/main/packages/media-parser"
   },
   "name": "@remotion/media-parser",
-  "version": "4.0.199",
+  "version": "4.0.200",
   "main": "dist/index.js",
   "sideEffects": false,
   "devDependencies": {
-    "@remotion/renderer": "4.0.199"
+    "@remotion/renderer": "4.0.200"
   },
   "publishConfig": {
     "access": "public"

package/src/boxes/iso-base-media/mdat/mdat.ts

@@ -91,7 +91,6 @@ export const parseMdat = async ({
 			await options.parserState.onAudioSample(sampleWithIndex.track.trackId, {
 				data: bytes,
 				timestamp: sampleWithIndex.samplePosition.offset,
-				offset: data.counter.getOffset(),
 				trackId: sampleWithIndex.track.trackId,
 				type: sampleWithIndex.samplePosition.isKeyframe ? 'key' : 'delta',
 			});

package/src/boxes/webm/ebml.ts

@@ -0,0 +1,78 @@
+// https://github.com/Vanilagy/webm-muxer/blob/main/src/ebml.ts#L101
+
+export const measureEBMLVarInt = (value: number) => {
+	if (value < (1 << 7) - 1) {
+		/** Top bit is set, leaving 7 bits to hold the integer, but we can't store
+		 * 127 because "all bits set to one" is a reserved value. Same thing for the
+		 * other cases below:
+		 */
+		return 1;
+	}
+
+	if (value < (1 << 14) - 1) {
+		return 2;
+	}
+
+	if (value < (1 << 21) - 1) {
+		return 3;
+	}
+
+	if (value < (1 << 28) - 1) {
+		return 4;
+	}
+
+	if (value < 2 ** 35 - 1) {
+		return 5;
+	}
+
+	if (value < 2 ** 42 - 1) {
+		return 6;
+	}
+
+	throw new Error('EBML VINT size not supported ' + value);
+};
+
+export const getVariableInt = (
+	value: number,
+	width: number = measureEBMLVarInt(value),
+) => {
+	switch (width) {
+		case 1:
+			return new Uint8Array([(1 << 7) | value]);
+		case 2:
+			return new Uint8Array([(1 << 6) | (value >> 8), value]);
+		case 3:
+			return new Uint8Array([(1 << 5) | (value >> 16), value >> 8, value]);
+		case 4:
+			return new Uint8Array([
+				(1 << 4) | (value >> 24),
+				value >> 16,
+				value >> 8,
+				value,
+			]);
+		case 5:
+			/**
+			 * JavaScript converts its doubles to 32-bit integers for bitwise
+			 * operations, so we need to do a division by 2^32 instead of a
+			 * right-shift of 32 to retain those top 3 bits
+			 */
+			return new Uint8Array([
+				(1 << 3) | ((value / 2 ** 32) & 0x7),
+				value >> 24,
+				value >> 16,
+				value >> 8,
+				value,
+			]);
+		case 6:
+			return new Uint8Array([
+				(1 << 2) | ((value / 2 ** 40) & 0x3),
+				(value / 2 ** 32) | 0,
+				value >> 24,
+				value >> 16,
+				value >> 8,
+				value,
+			]);
+		default:
+			throw new Error('Bad EBML VINT size ' + width);
+	}
+};

package/src/boxes/webm/make-header.ts

@@ -0,0 +1,48 @@
+import {getVariableInt} from './ebml';
+import {matroskaElements} from './segments/all-segments';
+
+export const webmPattern = new Uint8Array([0x1a, 0x45, 0xdf, 0xa3]);
+
+const matroskaToHex = (
+	matrId: (typeof matroskaElements)[keyof typeof matroskaElements],
+) => {
+	const numbers: number[] = [];
+	for (let i = 2; i < matrId.length; i += 2) {
+		const hex = matrId.substr(i, 2);
+		numbers.push(parseInt(hex, 16));
+	}
+
+	return numbers;
+};
+
+export const makeMatroskaHeader = () => {
+	const size = 0x23;
+
+	const array = new Uint8Array([
+		...webmPattern,
+		...getVariableInt(size),
+		...matroskaToHex(matroskaElements.EBMLVersion),
+		...getVariableInt(1),
+		1,
+		...matroskaToHex(matroskaElements.EBMLReadVersion),
+		...getVariableInt(1),
+		1,
+		...matroskaToHex(matroskaElements.EBMLMaxIDLength),
+		...getVariableInt(1),
+		4,
+		...matroskaToHex(matroskaElements.EBMLMaxSizeLength),
+		...getVariableInt(1),
+		8,
+		...matroskaToHex(matroskaElements.DocType),
+		...getVariableInt(8),
+		...new TextEncoder().encode('matroska'),
+		...matroskaToHex(matroskaElements.DocTypeVersion),
+		...getVariableInt(1),
+		4,
+		...matroskaToHex(matroskaElements.DocTypeReadVersion),
+		...getVariableInt(1),
+		2,
+	]);
+
+	return array;
+};

package/src/boxes/webm/segments/all-segments.ts

@@ -1,6 +1,11 @@
 export const matroskaElements = {
 	EBMLMaxIDLength: '0x42f2',
+	EBMLVersion: '0x4286',
+	EBMLReadVersion: '0x42F7',
 	EBMLMaxSizeLength: '0x42f3',
+	DocType: '0x4282',
+	DocTypeVersion: '0x4287',
+	DocTypeReadVersion: '0x4285',
 	Segment: '0x18538067',
 	SeekHead: '0x114d9b74',
 	Seek: '0x4dbb',

package/src/boxes/webm/segments/block-simple-block-flags.ts

@@ -0,0 +1,52 @@
+import type {BufferIterator} from '../../../buffer-iterator';
+import {matroskaElements} from './all-segments';
+
+type BlockFlags = {
+	invisible: boolean;
+	lacing: number;
+	keyframe: boolean | null;
+};
+
+export const parseBlockFlags = (
+	iterator: BufferIterator,
+	type:
+		| (typeof matroskaElements)['Block']
+		| (typeof matroskaElements)['SimpleBlock'],
+): BlockFlags => {
+	if (type === matroskaElements.Block) {
+		iterator.startReadingBits();
+		// Reserved
+		iterator.getBits(4);
+		const invisible = Boolean(iterator.getBits(1));
+		const lacing = iterator.getBits(2);
+		// unused
+		iterator.getBits(1);
+		iterator.stopReadingBits();
+		return {
+			invisible,
+			lacing,
+			keyframe: null,
+		};
+	}
+
+	if (type === matroskaElements.SimpleBlock) {
+		iterator.startReadingBits();
+
+		const keyframe = Boolean(iterator.getBits(1));
+		// Reserved
+		iterator.getBits(3);
+		const invisible = Boolean(iterator.getBits(1));
+		const lacing = iterator.getBits(2);
+		iterator.getBits(1);
+
+		iterator.stopReadingBits();
+
+		return {
+			invisible,
+			lacing,
+			keyframe,
+		};
+	}
+
+	throw new Error('Unexpected type');
+};

package/src/boxes/webm/segments/track-entry.ts

@@ -1,6 +1,9 @@
 import type {BufferIterator} from '../../../buffer-iterator';
 import type {ParserContext} from '../../../parser-context';
+import type {VideoSample} from '../../../webcodec-sample-types';
 import type {MatroskaSegment} from '../segments';
+import type {matroskaElements} from './all-segments';
+import {parseBlockFlags} from './block-simple-block-flags';
 import {expectChildren} from './parse-children';
 
 export type TrackEntrySegment = {
@@ -566,8 +569,18 @@ export const parseTimestampSegment = (
 	iterator: BufferIterator,
 	length: number,
 ): TimestampSegment => {
-	if (length > 2) {
-		throw new Error('Expected timestamp segment to be 1 byte or 2 bytes');
+	if (length > 3) {
+		throw new Error(
+			'Expected timestamp segment to be 1 byte or 2 bytes, but is ' + length,
+		);
+	}
+
+	if (length === 3) {
+		const val = iterator.getUint24();
+		return {
+			type: 'timestamp-segment',
+			timestamp: val,
+		};
 	}
 
 	const value = length === 2 ? iterator.getUint16() : iterator.getUint8();
@@ -578,66 +591,86 @@ export const parseTimestampSegment = (
 	};
 };
 
-export type SimpleBlockSegment = {
-	type: 'simple-block-segment';
+export type SimpleBlockOrBlockSegment = {
+	type: 'simple-block-or-block-segment';
 	length: number;
 	trackNumber: number;
 	timecode: number;
-	headerFlags: number;
-	keyframe: boolean;
-	lacing: [number, number];
+	keyframe: boolean | null;
+	lacing: number;
 	invisible: boolean;
-	children: MatroskaSegment[];
+	videoSample: Omit<VideoSample, 'type'> | null;
 };
 
 export type GetTracks = () => TrackEntrySegment[];
 
-export const parseSimpleBlockSegment = async ({
+export const parseSimpleBlockOrBlockSegment = async ({
 	iterator,
 	length,
 	parserContext,
+	type,
 }: {
 	iterator: BufferIterator;
 	length: number;
 	parserContext: ParserContext;
-}): Promise<SimpleBlockSegment> => {
+	type:
+		| (typeof matroskaElements)['Block']
+		| (typeof matroskaElements)['SimpleBlock'];
+}): Promise<SimpleBlockOrBlockSegment> => {
 	const start = iterator.counter.getOffset();
 	const trackNumber = iterator.getVint();
-	const timecode = iterator.getUint16();
-	const headerFlags = iterator.getUint8();
+	const timecodeRelativeToCluster = iterator.getUint16();
 
-	const invisible = Boolean((headerFlags >> 5) & 1);
-	const pos6 = (headerFlags >> 6) & 1;
-	const pos7 = (headerFlags >> 6) & 1;
-	const keyframe = Boolean((headerFlags >> 7) & 1);
+	const {invisible, lacing, keyframe} = parseBlockFlags(iterator, type);
 
 	const codec = parserContext.parserState.getTrackInfoByNumber(trackNumber);
+	const clusterOffset =
+		parserContext.parserState.getTimestampOffsetForByteOffset(
+			iterator.counter.getOffset(),
+		);
+
+	if (clusterOffset === undefined) {
+		throw new Error(
+			'Could not find offset for byte offset ' + iterator.counter.getOffset(),
+		);
+	}
+
+	const timecode = timecodeRelativeToCluster + clusterOffset;
 
 	if (!codec) {
 		throw new Error('Could not find codec for track ' + trackNumber);
 	}
 
-	const children: MatroskaSegment[] = [];
+	const remainingNow = length - (iterator.counter.getOffset() - start);
 
-	if (codec.codec.startsWith('V_')) {
-		const remainingNow = length - (iterator.counter.getOffset() - start);
+	let videoSample: Omit<VideoSample, 'type'> | null = null;
 
-		await parserContext.parserState.onVideoSample(trackNumber, {
+	if (codec.codec.startsWith('V_')) {
+		const partialVideoSample: Omit<VideoSample, 'type'> = {
 			data: iterator.getSlice(remainingNow),
 			cts: null,
 			dts: null,
 			duration: undefined,
-			type: keyframe ? 'key' : 'delta',
 			trackId: trackNumber,
 			timestamp: timecode,
-		});
+		};
+
+		if (keyframe === null) {
+			// If we don't know if this is a keyframe, we know after we emit the BlockGroup
+			videoSample = partialVideoSample;
+		} else {
+			const sample: VideoSample = {
+				...partialVideoSample,
+				type: keyframe ? 'key' : 'delta',
+			};
+
+			await parserContext.parserState.onVideoSample(trackNumber, sample);
+		}
 	}
 
 	if (codec.codec.startsWith('A_')) {
-		const vorbisRemaining = length - (iterator.counter.getOffset() - start);
 		await parserContext.parserState.onAudioSample(trackNumber, {
-			data: iterator.getSlice(vorbisRemaining),
-			offset: timecode,
+			data: iterator.getSlice(remainingNow),
 			trackId: trackNumber,
 			timestamp: timecode,
 			type: 'key',
@@ -650,15 +683,14 @@ export const parseSimpleBlockSegment = async ({
 	}
 
 	return {
-		type: 'simple-block-segment',
+		type: 'simple-block-or-block-segment',
 		length,
 		trackNumber,
 		timecode,
-		headerFlags,
 		keyframe,
-		lacing: [pos6, pos7],
+		lacing,
 		invisible,
-		children,
+		videoSample,
 	};
 };
 
@@ -705,6 +737,53 @@ export const parseBlockGroupSegment = async (
 	};
 };
 
+export type ReferenceBlockSegment = {
+	type: 'reference-block-segment';
+	referenceBlock: number;
+};
+
+export const parseReferenceBlockSegment = (
+	iterator: BufferIterator,
+	length: number,
+): ReferenceBlockSegment => {
+	if (length > 4) {
+		throw new Error(
+			`Expected reference block segment to be 4 bytes, but got ${length}`,
+		);
+	}
+
+	const referenceBlock =
+		length === 4
+			? iterator.getUint32()
+			: length === 3
+				? iterator.getUint24()
+				: length === 2
+					? iterator.getUint16()
+					: iterator.getUint8();
+
+	return {
+		type: 'reference-block-segment',
+		referenceBlock,
+	};
+};
+
+export type BlockAdditionsSegment = {
+	type: 'block-additions-segment';
+	blockAdditions: Uint8Array;
+};
+
+export const parseBlockAdditionsSegment = (
+	iterator: BufferIterator,
+	length: number,
+): BlockAdditionsSegment => {
+	const blockAdditions = iterator.getSlice(length);
+
+	return {
+		type: 'block-additions-segment',
+		blockAdditions,
+	};
+};
+
 export type BlockElement = {
 	type: 'block-element-segment';
 	length: number;

package/src/boxes/webm/segments.ts

@@ -2,6 +2,7 @@ import {registerTrack} from '../../add-new-matroska-tracks';
 import type {BufferIterator} from '../../buffer-iterator';
 import type {ParseResult} from '../../parse-result';
 import type {ParserContext} from '../../parser-context';
+import type {VideoSample} from '../../webcodec-sample-types';
 import {getTrack} from './get-track';
 import {matroskaElements} from './segments/all-segments';
 import type {DurationSegment} from './segments/duration';
@@ -29,6 +30,7 @@ import type {
 	AlphaModeSegment,
 	AudioSegment,
 	BitDepthSegment,
+	BlockAdditionsSegment,
 	BlockElement,
 	BlockGroupSegment,
 	ChannelsSegment,
@@ -46,9 +48,10 @@ import type {
 	InterlacedSegment,
 	LanguageSegment,
 	MaxBlockAdditionId,
+	ReferenceBlockSegment,
 	SamplingFrequencySegment,
 	SegmentUUIDSegment,
-	SimpleBlockSegment,
+	SimpleBlockOrBlockSegment,
 	TagSegment,
 	TagsSegment,
 	TimestampSegment,
@@ -64,6 +67,7 @@ import {
 	parseAlphaModeSegment,
 	parseAudioSegment,
 	parseBitDepthSegment,
+	parseBlockAdditionsSegment,
 	parseBlockElementSegment,
 	parseBlockGroupSegment,
 	parseChannelsSegment,
@@ -80,9 +84,10 @@ import {
 	parseInterlacedSegment,
 	parseLanguageSegment,
 	parseMaxBlockAdditionId,
+	parseReferenceBlockSegment,
 	parseSamplingFrequencySegment,
 	parseSegmentUUIDSegment,
-	parseSimpleBlockSegment,
+	parseSimpleBlockOrBlockSegment,
 	parseTagSegment,
 	parseTagsSegment,
 	parseTimestampSegment,
@@ -142,14 +147,16 @@ export type MatroskaSegment =
 	| TagSegment
 	| ClusterSegment
 	| TimestampSegment
-	| SimpleBlockSegment
+	| SimpleBlockOrBlockSegment
 	| BlockGroupSegment
 	| BlockElement
 	| SeekIdSegment
 	| AudioSegment
 	| SamplingFrequencySegment
 	| ChannelsSegment
-	| BitDepthSegment;
+	| BitDepthSegment
+	| ReferenceBlockSegment
+	| BlockAdditionsSegment;
 
 export type OnTrackEntrySegment = (trackEntry: TrackEntrySegment) => void;
 
@@ -360,20 +367,75 @@ const parseSegment = async ({
 		return parseBitDepthSegment(iterator, length);
 	}
 
-	if (segmentId === '0xe7') {
-		return parseTimestampSegment(iterator, length);
+	if (segmentId === matroskaElements.Timestamp) {
+		const offset = iterator.counter.getOffset();
+		const timestampSegment = parseTimestampSegment(iterator, length);
+
+		parserContext.parserState.setTimestampOffset(
+			offset,
+			timestampSegment.timestamp,
+		);
+
+		return timestampSegment;
 	}
 
-	if (segmentId === '0xa3') {
-		return parseSimpleBlockSegment({
+	if (
+		segmentId === matroskaElements.SimpleBlock ||
+		segmentId === matroskaElements.Block
+	) {
+		return parseSimpleBlockOrBlockSegment({
 			iterator,
 			length,
 			parserContext,
+			type: segmentId,
 		});
 	}
 
+	if (segmentId === matroskaElements.ReferenceBlock) {
+		return parseReferenceBlockSegment(iterator, length);
+	}
+
+	if (segmentId === matroskaElements.BlockAdditions) {
+		return parseBlockAdditionsSegment(iterator, length);
+	}
+
 	if (segmentId === '0xa0') {
-		return parseBlockGroupSegment(iterator, length, parserContext);
+		const blockGroup = await parseBlockGroupSegment(
+			iterator,
+			length,
+			parserContext,
+		);
+
+		// Blocks don't have information about keyframes.
+		// https://ffmpeg.org/pipermail/ffmpeg-devel/2015-June/173825.html
+		// "For Blocks, keyframes is
+		// inferred by the absence of ReferenceBlock element (as done by matroskadec).""
+
+		const block = blockGroup.children.find(
+			(c) => c.type === 'simple-block-or-block-segment',
+		);
+		if (!block || block.type !== 'simple-block-or-block-segment') {
+			throw new Error('Expected block segment');
+		}
+
+		const hasReferenceBlock = blockGroup.children.find(
+			(c) => c.type === 'reference-block-segment',
+		);
+
+		const partialVideoSample = block.videoSample;
+
+		if (partialVideoSample) {
+			const completeFrame: VideoSample = {
+				...partialVideoSample,
+				type: hasReferenceBlock ? 'delta' : 'key',
+			};
+			await parserContext.parserState.onVideoSample(
+				partialVideoSample.trackId,
+				completeFrame,
+			);
+		}
+
+		return blockGroup;
 	}
 
 	if (segmentId === '0xa1') {

package/src/buffer-iterator.ts

@@ -1,3 +1,4 @@
+import {webmPattern} from './boxes/webm/make-header';
 import {
 	knownIdsWithOneLength,
 	knownIdsWithThreeLength,
@@ -46,7 +47,6 @@ export class OffsetCounter {
 }
 
 const isoBaseMediaMp4Pattern = new TextEncoder().encode('ftyp');
-const webmPattern = new Uint8Array([0x1a, 0x45, 0xdf, 0xa3]);
 const mpegPattern = new Uint8Array([0xff, 0xf3, 0xe4, 0x64]);
 
 const matchesPattern = (pattern: Uint8Array) => {
@@ -388,6 +388,13 @@ export const getArrayBufferIterator = (
 			counter.increment(2);
 			return val;
 		},
+		getUint24: () => {
+			const val1 = view.getUint8(counter.getDiscardedOffset());
+			const val2 = view.getUint8(counter.getDiscardedOffset());
+			const val3 = view.getUint8(counter.getDiscardedOffset());
+			counter.increment(3);
+			return (val1 << 16) | (val2 << 8) | val3;
+		},
 
 		getInt16: () => {
 			const val = view.getInt16(counter.getDiscardedOffset());