@remotion/media-parser 4.0.192 → 4.0.194

package/src/get-audio-codec.ts

@@ -0,0 +1,143 @@
+/* eslint-disable max-depth */
+import type {Sample} from './boxes/iso-base-media/stsd/samples';
+import {trakBoxContainsAudio} from './get-fps';
+import type {KnownAudioCodecs} from './options';
+import type {AnySegment} from './parse-result';
+
+export const hasAudioCodec = (boxes: AnySegment[]): boolean => {
+	try {
+		return getAudioCodec(boxes) !== null;
+	} catch (e) {
+		return false;
+	}
+};
+
+const onEsdsBox = (child: AnySegment): KnownAudioCodecs | null => {
+	if (child && child.type === 'esds-box') {
+		const descriptor = child.descriptors.find(
+			(d) => d.type === 'decoder-config-descriptor',
+		);
+		if (descriptor && descriptor.type === 'decoder-config-descriptor') {
+			return descriptor.objectTypeIndication;
+		}
+	}
+
+	return null;
+};
+
+const onSample = (sample: Sample): KnownAudioCodecs | null | undefined => {
+	if (!sample) {
+		return null;
+	}
+
+	if (sample.type !== 'audio') {
+		return null;
+	}
+
+	if (sample.format === 'sowt') {
+		return 'aiff';
+	}
+
+	const child = sample.children.find((c) => c.type === 'esds-box');
+
+	if (child && child.type === 'esds-box') {
+		const ret = onEsdsBox(child);
+		if (ret) {
+			return ret;
+		}
+	}
+};
+
+export const getAudioCodec = (boxes: AnySegment[]): KnownAudioCodecs | null => {
+	const moovBox = boxes.find((b) => b.type === 'moov-box');
+	if (moovBox && moovBox.type === 'moov-box') {
+		const trakBox = moovBox.children.find(
+			(b) => b.type === 'trak-box' && trakBoxContainsAudio(b),
+		);
+		if (trakBox && trakBox.type === 'trak-box') {
+			const mdiaBox = trakBox.children.find(
+				(b) => b.type === 'regular-box' && b.boxType === 'mdia',
+			);
+			if (
+				mdiaBox &&
+				mdiaBox.type === 'regular-box' &&
+				mdiaBox.boxType === 'mdia'
+			) {
+				const minfBox = mdiaBox?.children.find(
+					(b) => b.type === 'regular-box' && b.boxType === 'minf',
+				);
+				if (
+					minfBox &&
+					minfBox.type === 'regular-box' &&
+					minfBox.boxType === 'minf'
+				) {
+					const stblBox = minfBox?.children.find(
+						(b) => b.type === 'regular-box' && b.boxType === 'stbl',
+					);
+					if (stblBox && stblBox.type === 'regular-box') {
+						const stsdBox = stblBox?.children.find(
+							(b) => b.type === 'stsd-box',
+						);
+						if (stsdBox && stsdBox.type === 'stsd-box') {
+							const sample = stsdBox.samples.find((s) => s.type === 'audio');
+							if (sample && sample.type === 'audio') {
+								const ret = onSample(sample);
+								if (ret) {
+									return ret;
+								}
+
+								const waveBox = sample.children.find(
+									(b) => b.type === 'regular-box' && b.boxType === 'wave',
+								);
+								if (
+									waveBox &&
+									waveBox.type === 'regular-box' &&
+									waveBox.boxType === 'wave'
+								) {
+									const esdsBox = waveBox.children.find(
+										(b) => b.type === 'esds-box',
+									);
+									if (esdsBox && esdsBox.type === 'esds-box') {
+										const ret2 = onEsdsBox(esdsBox);
+										if (ret2) {
+											return ret2;
+										}
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	const mainSegment = boxes.find((b) => b.type === 'main-segment');
+	if (!mainSegment || mainSegment.type !== 'main-segment') {
+		return null;
+	}
+
+	const tracksSegment = mainSegment.children.find(
+		(b) => b.type === 'tracks-segment',
+	);
+	if (!tracksSegment || tracksSegment.type !== 'tracks-segment') {
+		return null;
+	}
+
+	for (const track of tracksSegment.children) {
+		if (track.type === 'track-entry-segment') {
+			const trackType = track.children.find((b) => b.type === 'codec-segment');
+			if (trackType && trackType.type === 'codec-segment') {
+				if (trackType.codec === 'A_OPUS') {
+					return 'opus';
+				}
+
+				if (trackType.codec === 'A_PCM/INT/LIT') {
+					return 'pcm';
+				}
+			}
+		}
+	}
+
+	return null;
+};

package/src/get-dimensions.ts

@@ -14,10 +14,17 @@ const getDimensionsFromMatroska = (segments: MainSegment): Dimensions => {
 		throw new Error('No tracks segment');
 	}
 
-	// TODO: What if there are multiple video tracks, or audio track is first?
-	const trackEntrySegment = tracksSegment.children.find(
-		(b) => b.type === 'track-entry-segment',
-	);
+	const trackEntrySegment = tracksSegment.children.find((b) => {
+		if (b.type !== 'track-entry-segment') {
+			return false;
+		}
+
+		return (
+			b.children.find(
+				(c) => c.type === 'codec-segment' && c.codec.startsWith('V_'),
+			) !== undefined
+		);
+	});
 	if (!trackEntrySegment || trackEntrySegment.type !== 'track-entry-segment') {
 		throw new Error('No track entry segment');
 	}

package/src/get-fps.ts

@@ -27,6 +27,54 @@ type TimescaleAndDuration = {
 	duration: number;
 };
 
+export const trakBoxContainsAudio = (trakBox: AnySegment): boolean => {
+	if (trakBox.type !== 'trak-box') {
+		return false;
+	}
+
+	const {children} = trakBox;
+	const mediaBoxes = children.filter(
+		(c) => c.type === 'regular-box' && c.boxType === 'mdia',
+	);
+	if (!mediaBoxes || mediaBoxes.length === 0) {
+		return false;
+	}
+
+	const firstMediaBox = mediaBoxes[0];
+	if (
+		firstMediaBox.type !== 'regular-box' ||
+		firstMediaBox.boxType !== 'mdia'
+	) {
+		return false;
+	}
+
+	const minf = firstMediaBox.children.find(
+		(c) => c.type === 'regular-box' && c.boxType === 'minf',
+	);
+	if (!minf || minf.type !== 'regular-box' || minf.boxType !== 'minf') {
+		return false;
+	}
+
+	const stbl = minf.children.find(
+		(c) => c.type === 'regular-box' && c.boxType === 'stbl',
+	);
+	if (!stbl || stbl.type !== 'regular-box' || stbl.boxType !== 'stbl') {
+		return false;
+	}
+
+	const stsd = stbl.children.find((c) => c.type === 'stsd-box');
+	if (!stsd || stsd.type !== 'stsd-box') {
+		return false;
+	}
+
+	const videoSample = stsd.samples.find((s) => s.type === 'audio');
+	if (!videoSample || videoSample.type !== 'audio') {
+		return false;
+	}
+
+	return true;
+};
+
 export const trakBoxContainsVideo = (trakBox: AnySegment): boolean => {
 	if (trakBox.type !== 'trak-box') {
 		return false;

package/src/get-video-codec.ts

@@ -92,6 +92,10 @@ export const getVideoCodec = (boxes: AnySegment[]): KnownVideoCodecs | null => {
 				if (trackType.codec === 'V_AV1') {
 					return 'av1';
 				}
+
+				if (trackType.codec === 'V_MPEG4/ISO/AVC') {
+					return 'h264';
+				}
 			}
 		}
 	}

package/src/has-all-info.ts

@@ -1,3 +1,4 @@
+import {hasAudioCodec} from './get-audio-codec';
 import {hasDimensions} from './get-dimensions';
 import {hasDuration} from './get-duration';
 import {hasFps} from './get-fps';
@@ -6,12 +7,19 @@ import type {Options} from './options';
 import type {ParseResult} from './parse-result';
 
 export const hasAllInfo = (
-	options: Options<boolean, boolean, boolean, boolean, boolean>,
+	options: Options<boolean, boolean, boolean, boolean, boolean, boolean>,
 	parseResult: ParseResult,
 ) => {
 	const keys = Object.entries(options)
 		.filter(([, value]) => value)
-		.map(([key]) => key) as (keyof Options<true, true, true, true, true>)[];
+		.map(([key]) => key) as (keyof Options<
+		true,
+		true,
+		true,
+		true,
+		true,
+		true
+	>)[];
 
 	return keys.every((key) => {
 		if (key === 'boxes') {
@@ -34,6 +42,10 @@ export const hasAllInfo = (
 			return hasVideoCodec(parseResult.segments) !== null;
 		}
 
+		if (key === 'audioCodec') {
+			return hasAudioCodec(parseResult.segments) !== null;
+		}
+
 		throw new Error(`Unknown key: ${key satisfies never}`);
 	});
 };

package/src/options.ts

@@ -10,18 +10,28 @@ export type KnownVideoCodecs =
 	| 'av1'
 	| 'prores';
 
+export type KnownAudioCodecs =
+	| 'aac'
+	| 'mp3'
+	| 'aiff'
+	| 'opus'
+	| 'pcm'
+	| 'unknown';
+
 export type Options<
 	EnableDimensions extends boolean,
 	EnableDuration extends boolean,
 	EnableBoxes extends boolean,
 	EnableFps extends boolean,
 	EnableVideoCodec extends boolean,
+	EnableAudioCodec extends boolean,
 > = {
 	dimensions?: EnableDimensions;
 	durationInSeconds?: EnableDuration;
 	boxes?: EnableBoxes;
 	fps?: EnableFps;
 	videoCodec?: EnableVideoCodec;
+	audioCodec?: EnableAudioCodec;
 };
 
 export type Metadata<
@@ -30,11 +40,13 @@ export type Metadata<
 	EnableBoxes extends boolean,
 	EnableFps extends boolean,
 	EnableVideoCodec extends boolean,
+	EnableAudioCodec extends boolean,
 > = (EnableDimensions extends true ? {dimensions: Dimensions} : {}) &
 	(EnableDuration extends true ? {durationInSeconds: number | null} : {}) &
 	(EnableBoxes extends true ? {boxes: AnySegment[]} : {}) &
 	(EnableFps extends true ? {fps: number | null} : {}) &
-	(EnableVideoCodec extends true ? {videoCodec: KnownVideoCodecs | null} : {});
+	(EnableVideoCodec extends true ? {videoCodec: KnownVideoCodecs | null} : {}) &
+	(EnableAudioCodec extends true ? {audioCodec: KnownAudioCodecs | null} : {});
 
 export type ParseMedia = <
 	EnableDimensions extends boolean,
@@ -42,6 +54,7 @@ export type ParseMedia = <
 	EnableBoxes extends boolean,
 	EnableFps extends boolean,
 	EnableVideoCodec extends boolean,
+	EnableAudioCodec extends boolean,
 >(
 	src: string,
 	options: Options<
@@ -49,7 +62,8 @@ export type ParseMedia = <
 		EnableDuration,
 		EnableBoxes,
 		EnableFps,
-		EnableVideoCodec
+		EnableVideoCodec,
+		EnableAudioCodec
 	>,
 	readerInterface?: ReaderInterface,
 ) => Promise<
@@ -58,6 +72,7 @@ export type ParseMedia = <
 		EnableDuration,
 		EnableBoxes,
 		EnableFps,
-		EnableVideoCodec
+		EnableVideoCodec,
+		EnableAudioCodec
 	>
 >;

package/src/parse-media.ts

@@ -1,11 +1,14 @@
+import type {BufferIterator} from './buffer-iterator';
 import {getArrayBufferIterator} from './buffer-iterator';
 import {webReader} from './from-web';
+import {getAudioCodec} from './get-audio-codec';
 import {getDimensions} from './get-dimensions';
 import {getDuration} from './get-duration';
 import {getFps} from './get-fps';
 import {getVideoCodec} from './get-video-codec';
 import {hasAllInfo} from './has-all-info';
 import type {Metadata, ParseMedia} from './options';
+import type {ParseResult} from './parse-result';
 import {parseVideo} from './parse-video';
 
 export const parseMedia: ParseMedia = async (
@@ -13,29 +16,63 @@ export const parseMedia: ParseMedia = async (
 	options,
 	readerInterface = webReader,
 ) => {
-	const reader = await readerInterface.read(src, null);
+	const {reader, contentLength} = await readerInterface.read(src, null);
+	let currentReader = reader;
 
-	const returnValue = {} as Metadata<true, true, true, true, true>;
+	const returnValue = {} as Metadata<true, true, true, true, true, true>;
 
-	const iterator = getArrayBufferIterator(new Uint8Array([]));
-	let parseResult = parseVideo(iterator);
+	let iterator: BufferIterator | null = null;
+	let parseResult: ParseResult | null = null;
 
-	while (parseResult.status === 'incomplete') {
-		const result = await reader.read();
+	while (parseResult === null || parseResult.status === 'incomplete') {
+		const result = await currentReader.read();
 		if (result.done) {
 			break;
 		}
 
-		iterator.addData(result.value);
-		parseResult = parseResult.continueParsing();
+		if (iterator) {
+			iterator.addData(result.value);
+		} else {
+			iterator = getArrayBufferIterator(
+				result.value,
+				contentLength ?? undefined,
+			);
+		}
+
+		if (parseResult) {
+			parseResult = parseResult.continueParsing();
+		} else {
+			parseResult = parseVideo(iterator);
+		}
 
 		if (hasAllInfo(options, parseResult)) {
-			if (!reader.closed) {
-				reader.cancel(new Error('has all information'));
+			if (!currentReader.closed) {
+				currentReader.cancel(new Error('has all information'));
 			}
 
 			break;
 		}
+
+		if (
+			parseResult &&
+			parseResult.status === 'incomplete' &&
+			parseResult.skipTo !== null
+		) {
+			if (!currentReader.closed) {
+				currentReader.cancel(new Error('skipped ahead'));
+			}
+
+			const {reader: newReader} = await readerInterface.read(
+				src,
+				parseResult.skipTo,
+			);
+			currentReader = newReader;
+			iterator.skipTo(parseResult.skipTo);
+		}
+	}
+
+	if (!parseResult) {
+		throw new Error('Could not parse video');
 	}
 
 	if (options.dimensions) {
@@ -54,6 +91,10 @@ export const parseMedia: ParseMedia = async (
 		returnValue.videoCodec = getVideoCodec(parseResult.segments);
 	}
 
+	if (options.audioCodec) {
+		returnValue.audioCodec = getAudioCodec(parseResult.segments);
+	}
+
 	if (options.boxes) {
 		returnValue.boxes = parseResult.segments;
 	}

package/src/parse-result.ts

@@ -1,4 +1,5 @@
 import type {BaseBox} from './boxes/iso-base-media/base-type';
+import type {EsdsBox} from './boxes/iso-base-media/esds/esds';
 import type {FtypBox} from './boxes/iso-base-media/ftyp';
 import type {MdhdBox} from './boxes/iso-base-media/mdhd';
 import type {MoovBox} from './boxes/iso-base-media/moov/moov';
@@ -30,7 +31,8 @@ export type IsoBaseMediaBox =
 	| MoovBox
 	| TrakBox
 	| SttsBox
-	| MdhdBox;
+	| MdhdBox
+	| EsdsBox;
 
 export type AnySegment = MatroskaSegment | IsoBaseMediaBox;
 
@@ -42,5 +44,6 @@ export type ParseResult =
 	| {
 			status: 'incomplete';
 			segments: AnySegment[];
+			skipTo: number | null;
 			continueParsing: () => ParseResult;
 	  };

package/src/parse-video.ts

@@ -8,6 +8,7 @@ export type BoxAndNext =
 			type: 'complete';
 			box: IsoBaseMediaBox;
 			size: number;
+			skipTo: number | null;
 	  }
 	| {
 			type: 'incomplete';
@@ -21,6 +22,7 @@ export const parseVideo = (iterator: BufferIterator): ParseResult => {
 			continueParsing: () => {
 				return parseVideo(iterator);
 			},
+			skipTo: null,
 		};
 	}
 

package/src/reader.ts

@@ -1,7 +1,11 @@
+type ReadResult = {
+	reader: ReadableStreamDefaultReader<Uint8Array>;
+	contentLength: number | null;
+};
 type ReadContent = (
 	src: string,
-	range: [number, number] | null,
-) => Promise<ReadableStreamDefaultReader<Uint8Array>>;
+	range: [number, number] | number | null,
+) => Promise<ReadResult>;
 type GetLength = (src: string) => Promise<number>;
 
 export type ReaderInterface = {

package/src/test/matroska.test.ts

@@ -137,22 +137,21 @@ test('Should get duration of AV1 video', async () => {
 										{
 											type: 'color-segment',
 										},
-										{
-											id: '0x0163a294',
-											type: 'unknown-segment',
-										},
 									],
 								},
 								{
-									id: '0x0c000a0e',
-									type: 'unknown-segment',
+									codecPrivateData: [
+										129, 8, 12, 0, 10, 14, 0, 0, 0, 66, 171, 191, 195, 118, 0,
+										8, 8, 8, 8, 32,
+									],
+									type: 'codec-private-segment',
 								},
 							],
 						},
 					],
 				},
 				{
-					id: '0x',
+					id: '0x1254c367',
 					type: 'unknown-segment',
 				},
 			],

package/src/test/parse-esds.test.ts

@@ -0,0 +1,75 @@
+import {expect, test} from 'bun:test';
+import {parseEsds} from '../boxes/iso-base-media/esds/esds';
+import {getArrayBufferIterator} from '../buffer-iterator';
+
+test('Parse ESDS box', () => {
+	const buf = new Uint8Array([
+		// mock header
+		0, 0, 0, 0, 0, 0, 0, 0,
+		// actual box
+		0, 0, 0, 0, 3, 128, 128, 128, 27, 0, 2, 0, 4, 128, 128, 128, 13, 107, 21, 0,
+		0, 0, 0, 4, 226, 0, 0, 4, 226, 0, 6, 128, 128, 128, 1, 2,
+	]);
+
+	const iter = getArrayBufferIterator(buf);
+	iter.counter.increment(8);
+
+	expect(
+		parseEsds({
+			data: iter,
+			fileOffset: 8,
+			size: buf.length - 8,
+		}),
+	).toEqual({
+		type: 'esds-box',
+		version: 0,
+		tag: 3,
+		sizeOfInstance: 27,
+		esId: 2,
+		descriptors: [
+			{
+				type: 'decoder-config-descriptor',
+				objectTypeIndication: 'mp3',
+			},
+			{
+				type: 'sl-config-descriptor',
+			},
+		],
+	});
+});
+
+test('Parse two ESDS', () => {
+	const buf = new Uint8Array([
+		// mock header
+		0, 0, 0, 0, 0, 0, 0, 0,
+		// actual box
+		0, 0, 0, 0, 3, 25, 0, 0, 0, 4, 17, 64, 21, 0, 24, 0, 0, 4, 226, 0, 0, 4,
+		226, 0, 5, 2, 17, 144, 6, 1, 2, 0, 0, 0, 24, 115, 116, 116, 115,
+	]);
+
+	const iter = getArrayBufferIterator(buf);
+	iter.counter.increment(8);
+
+	expect(
+		parseEsds({
+			data: iter,
+			fileOffset: 8,
+			size: buf.length - 8,
+		}),
+	).toEqual({
+		type: 'esds-box',
+		version: 0,
+		tag: 3,
+		sizeOfInstance: 25,
+		esId: 0,
+		descriptors: [
+			{
+				objectTypeIndication: 'aac',
+				type: 'decoder-config-descriptor',
+			},
+			{
+				type: 'sl-config-descriptor',
+			},
+		],
+	});
+});