@remotion/media-parser 4.0.194 → 4.0.196

package/dist/webcodec-sample-types.d.ts

@@ -0,0 +1,21 @@
+import type { AudioTrack, VideoTrack } from './get-tracks';
+export type AudioSample = {
+    data: Uint8Array;
+    timestamp: number;
+    offset: number;
+    trackId: number;
+    type: 'key' | 'delta';
+};
+export type VideoSample = {
+    data: Uint8Array;
+    timestamp: number;
+    duration: number | undefined;
+    trackId: number;
+    type: 'key' | 'delta';
+    cts: number | null;
+    dts: number | null;
+};
+export type OnAudioSample = (sample: AudioSample) => void | Promise<void>;
+export type OnVideoSample = (sample: VideoSample) => void | Promise<void>;
+export type OnAudioTrack = (track: AudioTrack) => OnAudioSample | Promise<OnAudioSample | null> | null;
+export type OnVideoTrack = (track: VideoTrack) => OnVideoSample | Promise<OnVideoSample | null> | null;

package/dist/webcodec-sample-types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -3,11 +3,11 @@
     "url": "https://github.com/remotion-dev/remotion/tree/main/packages/media-parser"
   },
   "name": "@remotion/media-parser",
-  "version": "4.0.194",
+  "version": "4.0.196",
   "main": "dist/index.js",
   "sideEffects": false,
   "devDependencies": {
-    "@remotion/renderer": "4.0.194"
+    "@remotion/renderer": "4.0.196"
   },
   "publishConfig": {
     "access": "public"
@@ -18,12 +18,20 @@
   "exports": {
     ".": "./dist/index.js",
     "./node": "./dist/from-node.js",
+    "./fetch": "./dist/from-fetch.js",
+    "./web-file": "./dist/from-web-file.js",
     "./package.json": "./package.json"
   },
   "typesVersions": {
     ">=1.0": {
       "node": [
         "dist/from-node.d.ts"
+      ],
+      "fetch": [
+        "dist/from-fetch.d.ts"
+      ],
+      "web-file": [
+        "dist/from-web-file.d.ts"
       ]
     }
   },

package/src/add-new-matroska-tracks.ts

@@ -0,0 +1,23 @@
+import type {Track} from './get-tracks';
+import type {ParserContext} from './parser-context';
+import type {ParserState} from './parser-state';
+
+export const registerTrack = async ({
+	state,
+	options,
+	track,
+}: {
+	state: ParserState;
+	options: ParserContext;
+	track: Track;
+}) => {
+	if (track.type === 'video' && options.onVideoTrack) {
+		const callback = await options.onVideoTrack(track);
+		await state.registerVideoSampleCallback(track.trackId, callback ?? null);
+	}
+
+	if (track.type === 'audio' && options.onAudioTrack) {
+		const callback = await options.onAudioTrack(track);
+		await state.registerAudioSampleCallback(track.trackId, callback ?? null);
+	}
+};

package/src/boxes/iso-base-media/esds/decoder-specific-config.ts

@@ -0,0 +1,61 @@
+import type {BufferIterator} from '../../../buffer-iterator';
+
+type UnknownDecoderSpecificConfig = {
+	type: 'unknown-decoder-specific-config';
+};
+
+type AudioSpecificConfig = {
+	type: 'audio-specific-config';
+	audioObjectType: number;
+	samplingFrequencyIndex: number;
+	channelConfiguration: number;
+	asBytes: Uint8Array;
+};
+
+export type DecoderSpecificConfig =
+	| UnknownDecoderSpecificConfig
+	| AudioSpecificConfig;
+
+export const parseDecoderSpecificConfig = (
+	iterator: BufferIterator,
+): DecoderSpecificConfig => {
+	const layerTag = iterator.getUint8();
+	const layerSize = iterator.getPaddedFourByteNumber();
+
+	const start = iterator.counter.getOffset();
+
+	if (layerTag !== 5) {
+		iterator.discard(layerSize);
+		return {
+			type: 'unknown-decoder-specific-config',
+		};
+	}
+
+	// https://csclub.uwaterloo.ca/~pbarfuss/ISO14496-3-2009.pdf
+	// 1.6.2.1 AudioSpecificConfig
+
+	const bytes = iterator.getSlice(layerSize);
+	iterator.counter.decrement(layerSize);
+
+	iterator.startReadingBits();
+	const audioObjectType = iterator.getBits(5);
+	const samplingFrequencyIndex = iterator.getBits(4);
+	if (samplingFrequencyIndex === 0xf) {
+		iterator.getBits(24);
+	}
+
+	const channelConfiguration = iterator.getBits(4);
+	iterator.stopReadingBits();
+	const read = iterator.counter.getOffset() - start;
+	if (read < layerSize) {
+		iterator.discard(layerSize - read);
+	}
+
+	return {
+		type: 'audio-specific-config',
+		audioObjectType,
+		samplingFrequencyIndex,
+		channelConfiguration,
+		asBytes: bytes,
+	};
+};

package/src/boxes/iso-base-media/esds/esds-descriptors.ts

@@ -1,10 +1,20 @@
+/* eslint-disable no-constant-condition */
 import type {BufferIterator} from '../../../buffer-iterator';
+import type {DecoderSpecificConfig} from './decoder-specific-config';
+import {parseDecoderSpecificConfig} from './decoder-specific-config';
 
 type AudioObjectType = 'aac' | 'mp3' | 'unknown';
 
 type DecoderConfigDescriptor = {
 	type: 'decoder-config-descriptor';
 	objectTypeIndication: AudioObjectType;
+	asNumber: number;
+	streamType: number;
+	upStream: number;
+	bufferSizeDB: number;
+	maxBitrate: number;
+	avgBitrate: number;
+	decoderSpecificConfigs: DecoderSpecificConfig[];
 };
 
 const mapToObjectAudioIndicator = (num: number): AudioObjectType => {
@@ -51,13 +61,34 @@ export const processDescriptor = ({
 		const initialOffset = iterator.counter.getOffset();
 
 		const objectTypeIndication = iterator.getUint8();
+		iterator.startReadingBits();
+		const streamType = iterator.getBits(6);
+		const upStream = iterator.getBits(1);
+		// reserved
+		iterator.getBits(1);
+		const bufferSizeDB = iterator.getBits(24);
+		iterator.stopReadingBits();
+		const maxBitrate = iterator.getUint32();
+		const avgBitrate = iterator.getUint32();
+
+		const decoderSpecificConfigs: DecoderSpecificConfig[] = [];
+
+		while (size - (iterator.counter.getOffset() - initialOffset) >= 0) {
+			const decoderSpecificConfig = parseDecoderSpecificConfig(iterator);
+			decoderSpecificConfigs.push(decoderSpecificConfig);
+		}
 
-		const remaining = size - (iterator.counter.getOffset() - initialOffset);
-		iterator.discard(remaining);
 		return {
 			descriptor: {
 				type: 'decoder-config-descriptor',
 				objectTypeIndication: mapToObjectAudioIndicator(objectTypeIndication),
+				asNumber: objectTypeIndication,
+				bufferSizeDB,
+				streamType,
+				upStream,
+				avgBitrate,
+				maxBitrate,
+				decoderSpecificConfigs,
 			},
 		};
 	}

package/src/boxes/iso-base-media/make-track.ts

@@ -0,0 +1,157 @@
+import {
+	getAudioCodecStringFromTrak,
+	getNumberOfChannelsFromTrak,
+	getSampleRate,
+} from '../../get-audio-codec';
+import {
+	getTimescaleAndDuration,
+	trakBoxContainsAudio,
+	trakBoxContainsVideo,
+} from '../../get-fps';
+import {
+	applyAspectRatios,
+	applyTkhdBox,
+	getDisplayAspectRatio,
+	getSampleAspectRatio,
+	getVideoSample,
+} from '../../get-sample-aspect-ratio';
+import {getSamplePositions} from '../../get-sample-positions';
+import type {AudioTrack, OtherTrack, VideoTrack} from '../../get-tracks';
+import {getVideoCodecString} from '../../get-video-codec';
+import {
+	getCttsBox,
+	getStcoBox,
+	getStscBox,
+	getStssBox,
+	getStszBox,
+	getSttsBox,
+	getTkhdBox,
+	getVideoDescriptors,
+} from '../../traversal';
+import type {TrakBox} from './trak/trak';
+
+export const makeBaseMediaTrack = (
+	trakBox: TrakBox,
+): VideoTrack | AudioTrack | OtherTrack | null => {
+	const stszBox = getStszBox(trakBox);
+	const stcoBox = getStcoBox(trakBox);
+	const stscBox = getStscBox(trakBox);
+	const stssBox = getStssBox(trakBox);
+	const sttsBox = getSttsBox(trakBox);
+	const tkhdBox = getTkhdBox(trakBox);
+	const cttsBox = getCttsBox(trakBox);
+
+	const videoDescriptors = getVideoDescriptors(trakBox);
+	const timescaleAndDuration = getTimescaleAndDuration(trakBox);
+
+	if (!tkhdBox) {
+		throw new Error('Expected tkhd box in trak box');
+	}
+
+	if (!stszBox) {
+		throw new Error('Expected stsz box in trak box');
+	}
+
+	if (!stcoBox) {
+		throw new Error('Expected stco box in trak box');
+	}
+
+	if (!stscBox) {
+		throw new Error('Expected stsc box in trak box');
+	}
+
+	if (!sttsBox) {
+		throw new Error('Expected stts box in trak box');
+	}
+
+	if (!timescaleAndDuration) {
+		throw new Error('Expected timescale and duration in trak box');
+	}
+
+	const samplePositions = getSamplePositions({
+		stcoBox,
+		stscBox,
+		stszBox,
+		stssBox,
+		sttsBox,
+		cttsBox,
+	});
+
+	if (trakBoxContainsAudio(trakBox)) {
+		const numberOfChannels = getNumberOfChannelsFromTrak(trakBox);
+		if (numberOfChannels === null) {
+			throw new Error('Could not find number of channels');
+		}
+
+		const sampleRate = getSampleRate(trakBox);
+		if (sampleRate === null) {
+			throw new Error('Could not find sample rate');
+		}
+
+		const {codecString, description} = getAudioCodecStringFromTrak(trakBox);
+
+		return {
+			type: 'audio',
+			samplePositions,
+			trackId: tkhdBox.trackId,
+			timescale: timescaleAndDuration.timescale,
+			codec: codecString,
+			numberOfChannels,
+			sampleRate,
+			description,
+		};
+	}
+
+	if (!trakBoxContainsVideo(trakBox)) {
+		return {
+			type: 'other',
+			samplePositions,
+			trackId: tkhdBox.trackId,
+			timescale: timescaleAndDuration.timescale,
+		};
+	}
+
+	const videoSample = getVideoSample(trakBox);
+	if (!videoSample) {
+		throw new Error('No video sample');
+	}
+
+	const sampleAspectRatio = getSampleAspectRatio(trakBox);
+
+	const aspectRatioApplied = applyAspectRatios({
+		dimensions: videoSample,
+		sampleAspectRatio,
+		displayAspectRatio: getDisplayAspectRatio({
+			sampleAspectRatio,
+			nativeDimensions: videoSample,
+		}),
+	});
+
+	const {displayAspectHeight, displayAspectWidth, height, rotation, width} =
+		applyTkhdBox(aspectRatioApplied, tkhdBox);
+
+	const codec = getVideoCodecString(trakBox);
+
+	if (!codec) {
+		throw new Error('Could not find video codec');
+	}
+
+	const track: VideoTrack = {
+		type: 'video',
+		samplePositions,
+		trackId: tkhdBox.trackId,
+		description: videoDescriptors ?? undefined,
+		timescale: timescaleAndDuration.timescale,
+		codec,
+		sampleAspectRatio: getSampleAspectRatio(trakBox),
+		width,
+		height,
+		codedWidth: videoSample.width,
+		codedHeight: videoSample.height,
+		// Repeating those keys because they get picked up by VideoDecoder
+		displayAspectWidth,
+		displayAspectHeight,
+		rotation,
+	};
+	return track;
+};

package/src/boxes/iso-base-media/mdat/mdat.ts

@@ -0,0 +1,131 @@
+/* eslint-disable max-depth */
+import type {BufferIterator} from '../../../buffer-iterator';
+import {getTracks, hasTracks} from '../../../get-tracks';
+import type {AnySegment} from '../../../parse-result';
+import type {ParserContext} from '../../../parser-context';
+
+export interface MdatBox {
+	type: 'mdat-box';
+	samplesProcessed: boolean;
+	boxSize: number;
+	fileOffset: number;
+}
+
+export const parseMdat = async ({
+	data,
+	size,
+	fileOffset,
+	existingBoxes,
+	options,
+}: {
+	data: BufferIterator;
+	size: number;
+	fileOffset: number;
+	existingBoxes: AnySegment[];
+	options: ParserContext;
+}): Promise<MdatBox> => {
+	const alreadyHas = hasTracks(existingBoxes);
+	if (!alreadyHas) {
+		data.discard(size - 8);
+		return Promise.resolve({
+			type: 'mdat-box',
+			boxSize: size,
+			samplesProcessed: false,
+			fileOffset,
+		});
+	}
+
+	const tracks = getTracks(existingBoxes, options.parserState);
+	const allTracks = [
+		...tracks.videoTracks,
+		...tracks.audioTracks,
+		...tracks.otherTracks,
+	];
+
+	const flatSamples = allTracks
+		.map((track) => {
+			if (!track.samplePositions) {
+				throw new Error('No sample positions');
+			}
+
+			return track.samplePositions.map((samplePosition) => {
+				return {
+					track,
+					samplePosition,
+				};
+			});
+		})
+		.flat(1);
+
+	// eslint-disable-next-line no-constant-condition
+	while (true) {
+		const sampleWithIndex = flatSamples.find((sample) => {
+			return sample.samplePosition.offset === data.counter.getOffset();
+		});
+		if (!sampleWithIndex) {
+			// There are various reasons why in mdat we find weird stuff:
+			// - iphonevideo.hevc has a fake hoov atom which is not mapped
+			// - corrupted.mp4 has a corrupt table
+			const nextSample_ = flatSamples
+				.filter((s) => s.samplePosition.offset > data.counter.getOffset())
+				.sort((a, b) => a.samplePosition.offset - b.samplePosition.offset)[0];
+			if (nextSample_) {
+				data.discard(
+					nextSample_.samplePosition.offset - data.counter.getOffset(),
+				);
+				continue;
+			} else {
+				const bytesRemaining = size + fileOffset - data.counter.getOffset();
+				data.discard(bytesRemaining);
+				break;
+			}
+		}
+
+		if (data.bytesRemaining() < sampleWithIndex.samplePosition.size) {
+			break;
+		}
+
+		const bytes = data.getSlice(sampleWithIndex.samplePosition.size);
+
+		if (sampleWithIndex.track.type === 'audio') {
+			await options.parserState.onAudioSample(sampleWithIndex.track.trackId, {
+				data: bytes,
+				timestamp: sampleWithIndex.samplePosition.offset,
+				offset: data.counter.getOffset(),
+				trackId: sampleWithIndex.track.trackId,
+				type: sampleWithIndex.samplePosition.isKeyframe ? 'key' : 'delta',
+			});
+		}
+
+		if (sampleWithIndex.track.type === 'video') {
+			const timestamp =
+				(sampleWithIndex.samplePosition.cts * 1_000_000) /
+				sampleWithIndex.track.timescale;
+			const duration =
+				(sampleWithIndex.samplePosition.duration * 1_000_000) /
+				sampleWithIndex.track.timescale;
+
+			await options.parserState.onVideoSample(sampleWithIndex.track.trackId, {
+				data: bytes,
+				timestamp,
+				duration,
+				cts: sampleWithIndex.samplePosition.cts,
+				dts: sampleWithIndex.samplePosition.dts,
+				trackId: sampleWithIndex.track.trackId,
+				type: sampleWithIndex.samplePosition.isKeyframe ? 'key' : 'delta',
+			});
+		}
+
+		const remaining = size - (data.counter.getOffset() - fileOffset);
+		if (remaining === 0) {
+			break;
+		}
+	}
+
+	return Promise.resolve({
+		type: 'mdat-box',
+		boxSize: size,
+		samplesProcessed: true,
+		fileOffset,
+	});
+};

package/src/boxes/iso-base-media/moov/moov.ts

@@ -1,5 +1,6 @@
 import type {BufferIterator} from '../../../buffer-iterator';
 import type {AnySegment} from '../../../parse-result';
+import type {ParserContext} from '../../../parser-context';
 import type {BaseBox} from '../base-type';
 import {parseBoxes} from '../process-box';
 
@@ -8,20 +9,24 @@ export interface MoovBox extends BaseBox {
 	children: AnySegment[];
 }
 
-export const parseMoov = ({
+export const parseMoov = async ({
 	iterator,
 	offset,
 	size,
+	options,
 }: {
 	iterator: BufferIterator;
 	offset: number;
 	size: number;
-}): MoovBox => {
-	const children = parseBoxes({
+	options: ParserContext;
+}): Promise<MoovBox> => {
+	const children = await parseBoxes({
 		iterator,
 		maxBytes: size - (iterator.counter.getOffset() - offset),
 		allowIncompleteBoxes: false,
 		initialBoxes: [],
+		options,
+		continueMdat: false,
 	});
 
 	if (children.status === 'incomplete') {

package/src/boxes/iso-base-media/mvhd.ts

@@ -81,7 +81,11 @@ export const parseMvhd = ({
 	// matrix
 	const matrix: number[] = [];
 	for (let i = 0; i < 9; i++) {
-		matrix.push(iterator.getUint32());
+		if (i % 3 === 2) {
+			matrix.push(iterator.getFixedPointSigned230Number());
+		} else {
+			matrix.push(iterator.getFixedPointSigned1616Number());
+		}
 	}
 
 	// pre-defined
@@ -90,6 +94,8 @@ export const parseMvhd = ({
 	// next track id
 	const nextTrackId = iterator.getUint32();
 
+	volumeView.destroy();
+
 	return {
 		creationTime: toUnixTimestamp(creationTime),
 		modificationTime: toUnixTimestamp(modificationTime),