@remotion/media-parser 4.0.191 → 4.0.193

package/src/get-fps.ts

@@ -1,18 +1,183 @@
 import type {SttsBox} from './boxes/iso-base-media/stts/stts';
 import type {AnySegment} from './parse-result';
 
-const calculateFps = (sttsBox: SttsBox, timeScale: number) => {
-	let sum = 0;
+const calculateFps = ({
+	sttsBox,
+	timeScale,
+	durationInSamples,
+}: {
+	sttsBox: SttsBox;
+	timeScale: number;
+	durationInSamples: number;
+}) => {
 	let totalSamples = 0;
+
 	for (const sample of sttsBox.sampleDistribution) {
-		sum += sample.sampleCount * sample.sampleDelta;
 		totalSamples += sample.sampleCount;
 	}
 
-	return timeScale / (sum / totalSamples);
+	const durationInSeconds = durationInSamples / timeScale;
+	const fps = totalSamples / durationInSeconds;
+
+	return fps;
+};
+
+type TimescaleAndDuration = {
+	timescale: number;
+	duration: number;
+};
+
+export const trakBoxContainsAudio = (trakBox: AnySegment): boolean => {
+	if (trakBox.type !== 'trak-box') {
+		return false;
+	}
+
+	const {children} = trakBox;
+	const mediaBoxes = children.filter(
+		(c) => c.type === 'regular-box' && c.boxType === 'mdia',
+	);
+	if (!mediaBoxes || mediaBoxes.length === 0) {
+		return false;
+	}
+
+	const firstMediaBox = mediaBoxes[0];
+	if (
+		firstMediaBox.type !== 'regular-box' ||
+		firstMediaBox.boxType !== 'mdia'
+	) {
+		return false;
+	}
+
+	const minf = firstMediaBox.children.find(
+		(c) => c.type === 'regular-box' && c.boxType === 'minf',
+	);
+	if (!minf || minf.type !== 'regular-box' || minf.boxType !== 'minf') {
+		return false;
+	}
+
+	const stbl = minf.children.find(
+		(c) => c.type === 'regular-box' && c.boxType === 'stbl',
+	);
+	if (!stbl || stbl.type !== 'regular-box' || stbl.boxType !== 'stbl') {
+		return false;
+	}
+
+	const stsd = stbl.children.find((c) => c.type === 'stsd-box');
+	if (!stsd || stsd.type !== 'stsd-box') {
+		return false;
+	}
+
+	const videoSample = stsd.samples.find((s) => s.type === 'audio');
+	if (!videoSample || videoSample.type !== 'audio') {
+		return false;
+	}
+
+	return true;
+};
+
+export const trakBoxContainsVideo = (trakBox: AnySegment): boolean => {
+	if (trakBox.type !== 'trak-box') {
+		return false;
+	}
+
+	const {children} = trakBox;
+	const mediaBoxes = children.filter(
+		(c) => c.type === 'regular-box' && c.boxType === 'mdia',
+	);
+	if (!mediaBoxes || mediaBoxes.length === 0) {
+		return false;
+	}
+
+	const firstMediaBox = mediaBoxes[0];
+	if (
+		firstMediaBox.type !== 'regular-box' ||
+		firstMediaBox.boxType !== 'mdia'
+	) {
+		return false;
+	}
+
+	const minf = firstMediaBox.children.find(
+		(c) => c.type === 'regular-box' && c.boxType === 'minf',
+	);
+	if (!minf || minf.type !== 'regular-box' || minf.boxType !== 'minf') {
+		return false;
+	}
+
+	const stbl = minf.children.find(
+		(c) => c.type === 'regular-box' && c.boxType === 'stbl',
+	);
+	if (!stbl || stbl.type !== 'regular-box' || stbl.boxType !== 'stbl') {
+		return false;
+	}
+
+	const stsd = stbl.children.find((c) => c.type === 'stsd-box');
+	if (!stsd || stsd.type !== 'stsd-box') {
+		return false;
+	}
+
+	const videoSample = stsd.samples.find((s) => s.type === 'video');
+	if (!videoSample || videoSample.type !== 'video') {
+		return false;
+	}
+
+	return true;
+};
+
+export const getTimescaleAndDuration = (
+	boxes: AnySegment[],
+): TimescaleAndDuration | null => {
+	const moovBox = boxes.find((s) => s.type === 'moov-box');
+	if (!moovBox || moovBox.type !== 'moov-box') {
+		return null;
+	}
+
+	const {children} = moovBox;
+	const trackBoxes = children.filter((c) => c.type === 'trak-box');
+	if (!trackBoxes || trackBoxes.length === 0) {
+		return null;
+	}
+
+	const trackBox = trackBoxes.find(trakBoxContainsVideo);
+	if (!trackBox || trackBox.type !== 'trak-box') {
+		return null;
+	}
+
+	const trackBoxChildren = trackBox.children;
+	if (!trackBoxChildren || trackBoxChildren.length === 0) {
+		return null;
+	}
+
+	const mdiaBox = trackBoxChildren.find(
+		(c) => c.type === 'regular-box' && c.boxType === 'mdia',
+	);
+	if (
+		!mdiaBox ||
+		mdiaBox.type !== 'regular-box' ||
+		mdiaBox.boxType !== 'mdia'
+	) {
+		return null;
+	}
+
+	const mdhdBox = mdiaBox?.children.find((c) => c.type === 'mdhd-box');
+	if (mdhdBox && mdhdBox.type === 'mdhd-box') {
+		return {timescale: mdhdBox.timescale, duration: mdhdBox.duration};
+	}
+
+	const mvhdBox = moovBox.children.find((c) => c.type === 'mvhd-box');
+	if (!mvhdBox || mvhdBox.type !== 'mvhd-box') {
+		return null;
+	}
+
+	const {timeScale, durationInUnits} = mvhdBox;
+	return {timescale: timeScale, duration: durationInUnits};
 };
 
 export const getFps = (segments: AnySegment[]) => {
+	const timescaleAndDuration = getTimescaleAndDuration(segments);
+	if (!timescaleAndDuration) {
+		return null;
+	}
+
 	const moovBox = segments.find((s) => s.type === 'moov-box');
 	if (!moovBox || moovBox.type !== 'moov-box') {
 		return null;
@@ -23,16 +188,13 @@ export const getFps = (segments: AnySegment[]) => {
 		return null;
 	}
 
-	const {timeScale} = mvhdBox;
-
 	const {children} = moovBox;
 	const trackBoxes = children.filter((c) => c.type === 'trak-box');
 	if (!trackBoxes || trackBoxes.length === 0) {
 		return null;
 	}
 
-	// TODO: What if the video track is not the first track?
-	const trackBox = trackBoxes[0];
+	const trackBox = trackBoxes.find(trakBoxContainsVideo);
 	if (!trackBox || trackBox.type !== 'trak-box') {
 		return null;
 	}
@@ -80,7 +242,11 @@ export const getFps = (segments: AnySegment[]) => {
 		return null;
 	}
 
-	return calculateFps(sttsBox, timeScale);
+	return calculateFps({
+		sttsBox,
+		timeScale: timescaleAndDuration.timescale,
+		durationInSamples: timescaleAndDuration.duration,
+	});
 };
 
 export const hasFps = (boxes: AnySegment[]): boolean => {

package/src/get-video-codec.ts

@@ -0,0 +1,104 @@
+/* eslint-disable max-depth */
+import {trakBoxContainsVideo} from './get-fps';
+import type {KnownVideoCodecs} from './options';
+import type {AnySegment} from './parse-result';
+
+export const hasVideoCodec = (boxes: AnySegment[]): boolean => {
+	try {
+		return getVideoCodec(boxes) !== null;
+	} catch (e) {
+		return false;
+	}
+};
+
+export const getVideoCodec = (boxes: AnySegment[]): KnownVideoCodecs | null => {
+	const moovBox = boxes.find((b) => b.type === 'moov-box');
+	if (moovBox && moovBox.type === 'moov-box') {
+		const trakBox = moovBox.children.find(
+			(b) => b.type === 'trak-box' && trakBoxContainsVideo(b),
+		);
+		if (trakBox && trakBox.type === 'trak-box') {
+			const mdiaBox = trakBox.children.find(
+				(b) => b.type === 'regular-box' && b.boxType === 'mdia',
+			);
+			if (
+				mdiaBox &&
+				mdiaBox.type === 'regular-box' &&
+				mdiaBox.boxType === 'mdia'
+			) {
+				const minfBox = mdiaBox?.children.find(
+					(b) => b.type === 'regular-box' && b.boxType === 'minf',
+				);
+				if (
+					minfBox &&
+					minfBox.type === 'regular-box' &&
+					minfBox.boxType === 'minf'
+				) {
+					const stblBox = minfBox?.children.find(
+						(b) => b.type === 'regular-box' && b.boxType === 'stbl',
+					);
+					if (stblBox && stblBox.type === 'regular-box') {
+						const stsdBox = stblBox?.children.find(
+							(b) => b.type === 'stsd-box',
+						);
+						if (stsdBox && stsdBox.type === 'stsd-box') {
+							const videoSample = stsdBox.samples.find(
+								(s) => s.type === 'video',
+							);
+							if (videoSample && videoSample.type === 'video') {
+								if (videoSample.format === 'hvc1') {
+									return 'h265';
+								}
+
+								if (videoSample.format === 'avc1') {
+									return 'h264';
+								}
+
+								if (videoSample.format === 'ap4h') {
+									return 'prores';
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	const mainSegment = boxes.find((b) => b.type === 'main-segment');
+	if (!mainSegment || mainSegment.type !== 'main-segment') {
+		return null;
+	}
+
+	const tracksSegment = mainSegment.children.find(
+		(b) => b.type === 'tracks-segment',
+	);
+	if (!tracksSegment || tracksSegment.type !== 'tracks-segment') {
+		return null;
+	}
+
+	for (const track of tracksSegment.children) {
+		if (track.type === 'track-entry-segment') {
+			const trackType = track.children.find((b) => b.type === 'codec-segment');
+			if (trackType && trackType.type === 'codec-segment') {
+				if (trackType.codec === 'V_VP8') {
+					return 'vp8';
+				}
+
+				if (trackType.codec === 'V_VP9') {
+					return 'vp9';
+				}
+
+				if (trackType.codec === 'V_AV1') {
+					return 'av1';
+				}
+
+				if (trackType.codec === 'V_MPEG4/ISO/AVC') {
+					return 'h264';
+				}
+			}
+		}
+	}
+
+	return null;
+};

package/src/has-all-info.ts

@@ -1,16 +1,25 @@
+import {hasAudioCodec} from './get-audio-codec';
 import {hasDimensions} from './get-dimensions';
 import {hasDuration} from './get-duration';
 import {hasFps} from './get-fps';
+import {hasVideoCodec} from './get-video-codec';
 import type {Options} from './options';
 import type {ParseResult} from './parse-result';
 
 export const hasAllInfo = (
-	options: Options<boolean, boolean, boolean, boolean>,
+	options: Options<boolean, boolean, boolean, boolean, boolean, boolean>,
 	parseResult: ParseResult,
 ) => {
 	const keys = Object.entries(options)
 		.filter(([, value]) => value)
-		.map(([key]) => key) as (keyof Options<true, true, true, true>)[];
+		.map(([key]) => key) as (keyof Options<
+		true,
+		true,
+		true,
+		true,
+		true,
+		true
+	>)[];
 
 	return keys.every((key) => {
 		if (key === 'boxes') {
@@ -29,6 +38,14 @@ export const hasAllInfo = (
 			return hasFps(parseResult.segments) !== null;
 		}
 
+		if (key === 'videoCodec') {
+			return hasVideoCodec(parseResult.segments) !== null;
+		}
+
+		if (key === 'audioCodec') {
+			return hasAudioCodec(parseResult.segments) !== null;
+		}
+
 		throw new Error(`Unknown key: ${key satisfies never}`);
 	});
 };

package/src/options.ts

@@ -2,16 +2,36 @@ import type {Dimensions} from './get-dimensions';
 import type {AnySegment} from './parse-result';
 import type {ReaderInterface} from './reader';
 
+export type KnownVideoCodecs =
+	| 'h264'
+	| 'h265'
+	| 'vp8'
+	| 'vp9'
+	| 'av1'
+	| 'prores';
+
+export type KnownAudioCodecs =
+	| 'aac'
+	| 'mp3'
+	| 'aiff'
+	| 'opus'
+	| 'pcm'
+	| 'unknown';
+
 export type Options<
 	EnableDimensions extends boolean,
 	EnableDuration extends boolean,
 	EnableBoxes extends boolean,
 	EnableFps extends boolean,
+	EnableVideoCodec extends boolean,
+	EnableAudioCodec extends boolean,
 > = {
 	dimensions?: EnableDimensions;
 	durationInSeconds?: EnableDuration;
 	boxes?: EnableBoxes;
 	fps?: EnableFps;
+	videoCodec?: EnableVideoCodec;
+	audioCodec?: EnableAudioCodec;
 };
 
 export type Metadata<
@@ -19,20 +39,40 @@ export type Metadata<
 	EnableDuration extends boolean,
 	EnableBoxes extends boolean,
 	EnableFps extends boolean,
+	EnableVideoCodec extends boolean,
+	EnableAudioCodec extends boolean,
 > = (EnableDimensions extends true ? {dimensions: Dimensions} : {}) &
 	(EnableDuration extends true ? {durationInSeconds: number | null} : {}) &
 	(EnableBoxes extends true ? {boxes: AnySegment[]} : {}) &
-	(EnableFps extends true ? {fps: number | null} : {});
+	(EnableFps extends true ? {fps: number | null} : {}) &
+	(EnableVideoCodec extends true ? {videoCodec: KnownVideoCodecs | null} : {}) &
+	(EnableAudioCodec extends true ? {audioCodec: KnownAudioCodecs | null} : {});
 
 export type ParseMedia = <
 	EnableDimensions extends boolean,
 	EnableDuration extends boolean,
 	EnableBoxes extends boolean,
 	EnableFps extends boolean,
+	EnableVideoCodec extends boolean,
+	EnableAudioCodec extends boolean,
 >(
 	src: string,
-	options: Options<EnableDimensions, EnableDuration, EnableBoxes, EnableFps>,
+	options: Options<
+		EnableDimensions,
+		EnableDuration,
+		EnableBoxes,
+		EnableFps,
+		EnableVideoCodec,
+		EnableAudioCodec
+	>,
 	readerInterface?: ReaderInterface,
 ) => Promise<
-	Metadata<EnableDimensions, EnableDuration, EnableBoxes, EnableFps>
+	Metadata<
+		EnableDimensions,
+		EnableDuration,
+		EnableBoxes,
+		EnableFps,
+		EnableVideoCodec,
+		EnableAudioCodec
+	>
 >;

package/src/parse-media.ts

@@ -1,10 +1,14 @@
+import type {BufferIterator} from './buffer-iterator';
 import {getArrayBufferIterator} from './buffer-iterator';
 import {webReader} from './from-web';
+import {getAudioCodec} from './get-audio-codec';
 import {getDimensions} from './get-dimensions';
 import {getDuration} from './get-duration';
 import {getFps} from './get-fps';
+import {getVideoCodec} from './get-video-codec';
 import {hasAllInfo} from './has-all-info';
 import type {Metadata, ParseMedia} from './options';
+import type {ParseResult} from './parse-result';
 import {parseVideo} from './parse-video';
 
 export const parseMedia: ParseMedia = async (
@@ -12,21 +16,33 @@ export const parseMedia: ParseMedia = async (
 	options,
 	readerInterface = webReader,
 ) => {
-	const reader = await readerInterface.read(src, null);
+	const {reader, contentLength} = await readerInterface.read(src, null);
 
-	const returnValue = {} as Metadata<true, true, true, true>;
+	const returnValue = {} as Metadata<true, true, true, true, true, true>;
 
-	const iterator = getArrayBufferIterator(new Uint8Array([]));
-	let parseResult = parseVideo(iterator);
+	let iterator: BufferIterator | null = null;
+	let parseResult: ParseResult | null = null;
 
-	while (parseResult.status === 'incomplete') {
+	while (parseResult === null || parseResult.status === 'incomplete') {
 		const result = await reader.read();
 		if (result.done) {
 			break;
 		}
 
-		iterator.addData(result.value);
-		parseResult = parseResult.continueParsing();
+		if (iterator) {
+			iterator.addData(result.value);
+		} else {
+			iterator = getArrayBufferIterator(
+				result.value,
+				contentLength ?? undefined,
+			);
+		}
+
+		if (parseResult) {
+			parseResult = parseResult.continueParsing();
+		} else {
+			parseResult = parseVideo(iterator);
+		}
 
 		if (hasAllInfo(options, parseResult)) {
 			if (!reader.closed) {
@@ -37,6 +53,10 @@ export const parseMedia: ParseMedia = async (
 		}
 	}
 
+	if (!parseResult) {
+		throw new Error('Could not parse video');
+	}
+
 	if (options.dimensions) {
 		returnValue.dimensions = getDimensions(parseResult.segments);
 	}
@@ -49,6 +69,14 @@ export const parseMedia: ParseMedia = async (
 		returnValue.fps = getFps(parseResult.segments);
 	}
 
+	if (options.videoCodec) {
+		returnValue.videoCodec = getVideoCodec(parseResult.segments);
+	}
+
+	if (options.audioCodec) {
+		returnValue.audioCodec = getAudioCodec(parseResult.segments);
+	}
+
 	if (options.boxes) {
 		returnValue.boxes = parseResult.segments;
 	}

package/src/parse-result.ts

@@ -1,5 +1,7 @@
 import type {BaseBox} from './boxes/iso-base-media/base-type';
+import type {EsdsBox} from './boxes/iso-base-media/esds/esds';
 import type {FtypBox} from './boxes/iso-base-media/ftyp';
+import type {MdhdBox} from './boxes/iso-base-media/mdhd';
 import type {MoovBox} from './boxes/iso-base-media/moov/moov';
 import type {MvhdBox} from './boxes/iso-base-media/mvhd';
 import type {KeysBox} from './boxes/iso-base-media/stsd/keys';
@@ -28,7 +30,9 @@ export type IsoBaseMediaBox =
 	| KeysBox
 	| MoovBox
 	| TrakBox
-	| SttsBox;
+	| SttsBox
+	| MdhdBox
+	| EsdsBox;
 
 export type AnySegment = MatroskaSegment | IsoBaseMediaBox;
 

package/src/reader.ts

@@ -1,7 +1,11 @@
+type ReadResult = {
+	reader: ReadableStreamDefaultReader<Uint8Array>;
+	contentLength: number | null;
+};
 type ReadContent = (
 	src: string,
 	range: [number, number] | null,
-) => Promise<ReadableStreamDefaultReader<Uint8Array>>;
+) => Promise<ReadResult>;
 type GetLength = (src: string) => Promise<number>;
 
 export type ReaderInterface = {

package/src/test/matroska.test.ts

@@ -140,19 +140,18 @@ test('Should get duration of AV1 video', async () => {
 									],
 								},
 								{
-									id: '0x0163a294',
-									type: 'unknown-segment',
+									codecPrivateData: [
+										129, 8, 12, 0, 10, 14, 0, 0, 0, 66, 171, 191, 195, 118, 0,
+										8, 8, 8, 8, 32,
+									],
+									type: 'codec-private-segment',
 								},
 							],
 						},
-						{
-							id: '0x0c000a0e',
-							type: 'unknown-segment',
-						},
 					],
 				},
 				{
-					id: '0x',
+					id: '0x1254c367',
 					type: 'unknown-segment',
 				},
 			],

package/src/test/parse-esds.test.ts

@@ -0,0 +1,75 @@
+import {expect, test} from 'bun:test';
+import {parseEsds} from '../boxes/iso-base-media/esds/esds';
+import {getArrayBufferIterator} from '../buffer-iterator';
+
+test('Parse ESDS box', () => {
+	const buf = new Uint8Array([
+		// mock header
+		0, 0, 0, 0, 0, 0, 0, 0,
+		// actual box
+		0, 0, 0, 0, 3, 128, 128, 128, 27, 0, 2, 0, 4, 128, 128, 128, 13, 107, 21, 0,
+		0, 0, 0, 4, 226, 0, 0, 4, 226, 0, 6, 128, 128, 128, 1, 2,
+	]);
+
+	const iter = getArrayBufferIterator(buf);
+	iter.counter.increment(8);
+
+	expect(
+		parseEsds({
+			data: iter,
+			fileOffset: 8,
+			size: buf.length - 8,
+		}),
+	).toEqual({
+		type: 'esds-box',
+		version: 0,
+		tag: 3,
+		sizeOfInstance: 27,
+		esId: 2,
+		descriptors: [
+			{
+				type: 'decoder-config-descriptor',
+				objectTypeIndication: 'mp3',
+			},
+			{
+				type: 'sl-config-descriptor',
+			},
+		],
+	});
+});
+
+test('Parse two ESDS', () => {
+	const buf = new Uint8Array([
+		// mock header
+		0, 0, 0, 0, 0, 0, 0, 0,
+		// actual box
+		0, 0, 0, 0, 3, 25, 0, 0, 0, 4, 17, 64, 21, 0, 24, 0, 0, 4, 226, 0, 0, 4,
+		226, 0, 5, 2, 17, 144, 6, 1, 2, 0, 0, 0, 24, 115, 116, 116, 115,
+	]);
+
+	const iter = getArrayBufferIterator(buf);
+	iter.counter.increment(8);
+
+	expect(
+		parseEsds({
+			data: iter,
+			fileOffset: 8,
+			size: buf.length - 8,
+		}),
+	).toEqual({
+		type: 'esds-box',
+		version: 0,
+		tag: 3,
+		sizeOfInstance: 25,
+		esId: 0,
+		descriptors: [
+			{
+				objectTypeIndication: 'aac',
+				type: 'decoder-config-descriptor',
+			},
+			{
+				type: 'sl-config-descriptor',
+			},
+		],
+	});
+});

package/src/test/parse-webm.test.ts

@@ -8,8 +8,10 @@ test('should be able to parse a WebM', async () => {
 		RenderInternals.exampleVideos.transparentWebm,
 		{
 			durationInSeconds: true,
+			videoCodec: true,
 		},
 		nodeReader,
 	);
 	expect(webm.durationInSeconds).toBe(5);
+	expect(webm.videoCodec).toBe('vp8');
 });