@editframe/assets 0.18.21-beta.0 → 0.18.22-beta.0

package/dist/generateFragmentIndex.js

@@ -1,8 +1,25 @@
+import { PacketProbe } from "./Probe.js";
 import debug from "debug";
-import { Transform, Writable } from "node:stream";
+import { Readable, Transform, Writable } from "node:stream";
 import { pipeline } from "node:stream/promises";
-import { EncodedPacketSink, Input, MP4, StreamSource } from "mediabunny";
 const log = debug("ef:generateFragmentIndex");
+function constructH264CodecString(codecTagString, profile, level) {
+	if (codecTagString !== "avc1" || !profile || level === void 0) return codecTagString;
+	const profileMap = {
+		"Baseline": 66,
+		"Main": 77,
+		"High": 100,
+		"High 10": 110,
+		"High 422": 122,
+		"High 444": 244
+	};
+	const profileIdc = profileMap[profile];
+	if (!profileIdc) return codecTagString;
+	const profileHex = profileIdc.toString(16).padStart(2, "0");
+	const constraintFlags = "00";
+	const levelHex = level.toString(16).padStart(2, "0");
+	return `${codecTagString}.${profileHex}${constraintFlags}${levelHex}`;
+}
 /**
 * Streaming MP4 box parser that detects box boundaries without loading entire file into memory
 */
@@ -61,12 +78,7 @@ var StreamingBoxParser = class extends Transform {
 						mdatOffset: box.offset
 					});
 					this.currentMoof = null;
-				} else this.fragments.push({
-					type: "media",
-					offset: box.offset,
-					size: box.size,
-					mdatOffset: box.offset
-				});
+				} else log(`Found non-fragmented mdat at offset ${box.offset}, skipping for fragment index`);
 				break;
 		}
 	}
@@ -83,34 +95,18 @@ var StreamingBoxParser = class extends Transform {
 		return this.fragments;
 	}
 };
-function extractFragmentData(chunks, initFragment, mediaFragment) {
-	const extractBytes = (offset, size) => {
-		const buffer = Buffer.alloc(size);
-		let written = 0;
-		let currentOffset = 0;
-		for (const chunk of chunks) {
-			if (currentOffset + chunk.length <= offset) {
-				currentOffset += chunk.length;
-				continue;
-			}
-			if (currentOffset >= offset + size) break;
-			const chunkStart = Math.max(0, offset - currentOffset);
-			const chunkEnd = Math.min(chunk.length, offset + size - currentOffset);
-			const copySize = chunkEnd - chunkStart;
-			chunk.copy(buffer, written, chunkStart, chunkEnd);
-			written += copySize;
-			currentOffset += chunk.length;
-			if (written >= size) break;
+function createFragmentStream(fragmentData) {
+	let offset = 0;
+	return new Readable({ read() {
+		if (offset >= fragmentData.length) {
+			this.push(null);
+			return;
 		}
-		return new Uint8Array(buffer.buffer, buffer.byteOffset, written);
-	};
-	if (!initFragment) return extractBytes(mediaFragment.offset, mediaFragment.size);
-	const initData = extractBytes(initFragment.offset, initFragment.size);
-	const mediaData = extractBytes(mediaFragment.offset, mediaFragment.size);
-	const combined = new Uint8Array(initData.length + mediaData.length);
-	combined.set(initData, 0);
-	combined.set(mediaData, initData.length);
-	return combined;
+		const chunkSize = Math.min(64 * 1024, fragmentData.length - offset);
+		const chunk = fragmentData.slice(offset, offset + chunkSize);
+		offset += chunkSize;
+		this.push(Buffer.from(chunk));
+	} });
 }
 const generateFragmentIndex = async (inputStream, startTimeOffsetMs, trackIdMapping) => {
 	const parser = new StreamingBoxParser();
@@ -124,166 +120,123 @@ const generateFragmentIndex = async (inputStream, startTimeOffsetMs, trackIdMapp
 	await pipeline(inputStream, parser, dest);
 	const fragments = parser.getFragments();
 	if (totalSize === 0) return {};
-	const source = new StreamSource({
-		read: async (start, end) => {
-			const size = end - start;
-			const buffer = Buffer.alloc(size);
-			let written = 0;
-			let currentOffset = 0;
-			for (const chunk of chunks) {
-				if (currentOffset + chunk.length <= start) {
-					currentOffset += chunk.length;
-					continue;
-				}
-				if (currentOffset >= end) break;
-				const chunkStart = Math.max(0, start - currentOffset);
-				const chunkEnd = Math.min(chunk.length, end - currentOffset);
-				const copySize = chunkEnd - chunkStart;
-				chunk.copy(buffer, written, chunkStart, chunkEnd);
-				written += copySize;
-				currentOffset += chunk.length;
-				if (written >= size) break;
-			}
-			return new Uint8Array(buffer.buffer, buffer.byteOffset, written);
-		},
-		getSize: async () => totalSize
-	});
-	let input;
-	let videoTracks;
-	let audioTracks;
+	const completeData = Buffer.concat(chunks);
+	const completeStream = createFragmentStream(new Uint8Array(completeData.buffer, completeData.byteOffset, completeData.byteLength));
+	let probe;
 	try {
-		input = new Input({
-			formats: [MP4],
-			source
-		});
-		videoTracks = await input.getVideoTracks();
-		audioTracks = await input.getAudioTracks();
+		probe = await PacketProbe.probeStream(completeStream);
 	} catch (error) {
-		console.warn("Failed to parse with parser:", error);
+		console.warn("Failed to probe stream with ffprobe:", error);
 		return {};
 	}
+	const videoStreams = probe.videoStreams;
+	const audioStreams = probe.audioStreams;
 	const trackIndexes = {};
 	const initFragment = fragments.find((f) => f.type === "init");
 	const mediaFragments = fragments.filter((f) => f.type === "media");
-	const videoFragmentTimings = [];
-	const audioFragmentTimings = [];
+	const fragmentTimingData = [];
 	for (let fragmentIndex = 0; fragmentIndex < mediaFragments.length; fragmentIndex++) {
 		const fragment = mediaFragments[fragmentIndex];
-		const fragmentData = extractFragmentData(chunks, initFragment, fragment);
-		const fragmentSource = new StreamSource({
-			read: async (start, end) => fragmentData.subarray(start, end),
-			getSize: async () => fragmentData.length
+		const fragmentStart = fragment.offset;
+		const fragmentEnd = fragment.offset + fragment.size;
+		const videoPackets = probe.packets.filter((packet) => {
+			const stream = videoStreams.find((s) => s.index === packet.stream_index);
+			return stream?.codec_type === "video" && packet.pos !== void 0 && packet.pos >= fragmentStart && packet.pos < fragmentEnd;
+		}).map((packet) => ({
+			pts: packet.pts,
+			dts: packet.dts,
+			duration: packet.duration,
+			isKeyframe: packet.flags?.includes("K") ?? false
+		}));
+		const audioPackets = probe.packets.filter((packet) => {
+			const stream = audioStreams.find((s) => s.index === packet.stream_index);
+			return stream?.codec_type === "audio" && packet.pos !== void 0 && packet.pos >= fragmentStart && packet.pos < fragmentEnd;
+		}).map((packet) => ({
+			pts: packet.pts,
+			dts: packet.dts,
+			duration: packet.duration
+		}));
+		fragmentTimingData.push({
+			fragmentIndex,
+			videoPackets,
+			audioPackets
 		});
-		try {
-			const fragmentInput = new Input({
-				formats: [MP4],
-				source: fragmentSource
-			});
-			const fragmentVideoTracks = await fragmentInput.getVideoTracks();
-			if (fragmentVideoTracks.length > 0) {
-				const track = fragmentVideoTracks[0];
-				const sink = new EncodedPacketSink(track);
-				const packets = [];
-				for await (const packet of sink.packets()) packets.push({
-					timestamp: packet.timestamp,
-					duration: packet.duration
-				});
-				if (packets.length > 0) {
-					const firstPacket = packets[0];
-					const lastPacket = packets[packets.length - 1];
-					const actualDuration = lastPacket.timestamp + lastPacket.duration - firstPacket.timestamp;
-					videoFragmentTimings.push({
-						fragmentIndex,
-						cts: Math.round(firstPacket.timestamp * track.timeResolution),
-						dts: Math.round(firstPacket.timestamp * track.timeResolution),
-						duration: Math.round(actualDuration * track.timeResolution),
-						sampleCount: packets.length,
-						timescale: track.timeResolution
-					});
-				}
-			}
-			const fragmentAudioTracks = await fragmentInput.getAudioTracks();
-			if (fragmentAudioTracks.length > 0) {
-				const track = fragmentAudioTracks[0];
-				const sink = new EncodedPacketSink(track);
-				const packets = [];
-				for await (const packet of sink.packets()) packets.push({
-					timestamp: packet.timestamp,
-					duration: packet.duration
-				});
-				if (packets.length > 0) {
-					const firstPacket = packets[0];
-					const lastPacket = packets[packets.length - 1];
-					const actualDuration = lastPacket.timestamp + lastPacket.duration - firstPacket.timestamp;
-					audioFragmentTimings.push({
-						fragmentIndex,
-						cts: Math.round(firstPacket.timestamp * track.timeResolution),
-						dts: Math.round(firstPacket.timestamp * track.timeResolution),
-						duration: Math.round(actualDuration * track.timeResolution),
-						sampleCount: packets.length,
-						timescale: track.timeResolution
-					});
-				}
-			}
-		} catch (error) {
-			console.warn(`Failed to parse fragment ${fragmentIndex}:`, error);
-			continue;
-		}
 	}
-	for (const track of videoTracks) {
-		const sink = new EncodedPacketSink(track);
+	for (const videoStream of videoStreams) {
 		const segments = [];
-		let sampleCount = 0;
-		let totalDuration = 0;
-		const allPackets = [];
-		for await (const packet of sink.packets()) {
-			allPackets.push({
-				timestamp: packet.timestamp,
-				duration: packet.duration
-			});
-			sampleCount++;
+		const totalVideoPackets = probe.packets.filter((p) => p.stream_index === videoStream.index);
+		const keyframePackets = totalVideoPackets.filter((p) => p.flags?.includes("K"));
+		const totalSampleCount = keyframePackets.length;
+		log(`Complete stream has ${totalVideoPackets.length} video packets, ${keyframePackets.length} keyframes for stream ${videoStream.index}`);
+		const timebase = probe.videoTimebase;
+		if (!timebase) {
+			console.warn("No timebase found for video stream");
+			continue;
 		}
+		const timescale = Math.round(timebase.den / timebase.num);
 		let trackStartTimeOffsetMs;
-		if (allPackets.length > 0) {
-			const firstPacketTime = allPackets[0].timestamp;
-			if (Math.abs(firstPacketTime) > .01) trackStartTimeOffsetMs = firstPacketTime * 1e3;
+		const allVideoPackets = probe.packets.filter((p) => p.stream_index === videoStream.index);
+		if (allVideoPackets.length > 0) {
+			const firstPacketTime = allVideoPackets[0].dts_time;
+			log(`First video packet dts_time: ${firstPacketTime}, pts_time: ${allVideoPackets[0].pts_time}`);
+			const presentationTime = allVideoPackets[0].pts_time;
+			if (Math.abs(presentationTime) > .01) trackStartTimeOffsetMs = presentationTime * 1e3;
 		}
 		if (startTimeOffsetMs !== void 0) trackStartTimeOffsetMs = startTimeOffsetMs;
-		const timescale = Math.round(track.timeResolution);
-		let accumulatedDts = 0;
-		let accumulatedCts = 0;
-		for (const timing of videoFragmentTimings) {
-			const fragment = mediaFragments[timing.fragmentIndex];
+		log(`Processing ${fragmentTimingData.length} fragments for video stream ${videoStream.index}`);
+		for (const fragmentData of fragmentTimingData) {
+			const fragment = mediaFragments[fragmentData.fragmentIndex];
+			const videoPackets = fragmentData.videoPackets;
+			log(`Fragment ${fragmentData.fragmentIndex}: ${videoPackets.length} video packets`);
+			if (videoPackets.length === 0) {
+				log(`Skipping fragment ${fragmentData.fragmentIndex} - no video packets`);
+				continue;
+			}
+			const firstPacket = videoPackets[0];
+			const keyframe = videoPackets.find((p) => p.isKeyframe) || firstPacket;
+			const segmentCts = Math.round(keyframe.pts * timescale / timebase.den);
+			const segmentDts = Math.round(keyframe.dts * timescale / timebase.den);
+			const nextFragmentData = fragmentTimingData[fragmentData.fragmentIndex + 1];
+			const nextKeyframe = nextFragmentData?.videoPackets.find((p) => p.isKeyframe);
+			let segmentDuration;
+			if (nextKeyframe) {
+				const nextSegmentCts = Math.round(nextKeyframe.pts * timescale / timebase.den);
+				segmentDuration = nextSegmentCts - segmentCts;
+			} else {
+				const allVideoPackets$1 = probe.packets.filter((p) => {
+					const stream = videoStreams.find((s) => s.index === p.stream_index);
+					return stream?.codec_type === "video";
+				}).sort((a, b) => a.pts - b.pts);
+				const lastPacket = allVideoPackets$1[allVideoPackets$1.length - 1];
+				const streamEnd = Math.round((lastPacket.pts + (lastPacket.duration || 0)) * timescale / timebase.den);
+				segmentDuration = streamEnd - segmentCts;
+			}
 			segments.push({
-				cts: accumulatedCts,
-				dts: accumulatedDts,
-				duration: timing.duration,
+				cts: segmentCts,
+				dts: segmentDts,
+				duration: segmentDuration,
 				offset: fragment.offset,
 				size: fragment.size
 			});
-			accumulatedDts += timing.duration;
-			accumulatedCts += timing.duration;
-			totalDuration += timing.duration / timescale;
 		}
-		let width = 1920;
-		let height = 1080;
-		try {
-			const decoderConfig = await track.getDecoderConfig();
-			if (decoderConfig) {
-				width = decoderConfig.codedWidth ?? decoderConfig.width ?? width;
-				height = decoderConfig.codedHeight ?? decoderConfig.height ?? height;
-			}
-		} catch (e) {}
-		const finalTrackId = trackIdMapping?.[track.id] ?? track.id;
+		let totalDuration = 0;
+		if (totalVideoPackets.length > 0) {
+			const firstPacket = totalVideoPackets[0];
+			const lastPacket = totalVideoPackets[totalVideoPackets.length - 1];
+			const firstPts = Math.round(firstPacket.pts * timescale / timebase.den);
+			const lastPts = Math.round(lastPacket.pts * timescale / timebase.den);
+			totalDuration = lastPts - firstPts;
+		}
+		const finalTrackId = trackIdMapping?.[videoStream.index] ?? videoStream.index + 1;
 		trackIndexes[finalTrackId] = {
 			track: finalTrackId,
 			type: "video",
-			width,
-			height,
-			timescale: Math.round(track.timeResolution),
-			sample_count: sampleCount,
-			codec: await track.getCodecParameterString() || "",
-			duration: Math.round(totalDuration * track.timeResolution),
+			width: videoStream.coded_width || videoStream.width,
+			height: videoStream.coded_height || videoStream.height,
+			timescale,
+			sample_count: totalSampleCount,
+			codec: constructH264CodecString(videoStream.codec_tag_string, videoStream.profile, videoStream.level),
+			duration: totalDuration,
 			startTimeOffsetMs: trackStartTimeOffsetMs,
 			initSegment: {
 				offset: 0,
@@ -292,52 +245,70 @@ const generateFragmentIndex = async (inputStream, startTimeOffsetMs, trackIdMapp
 			segments
 		};
 	}
-	for (const track of audioTracks) {
-		const sink = new EncodedPacketSink(track);
+	for (const audioStream of audioStreams) {
 		const segments = [];
-		let sampleCount = 0;
-		let totalDuration = 0;
-		const allPackets = [];
-		for await (const packet of sink.packets()) {
-			allPackets.push({
-				timestamp: packet.timestamp,
-				duration: packet.duration
-			});
-			sampleCount++;
+		const totalAudioPackets = probe.packets.filter((p) => p.stream_index === audioStream.index);
+		const totalSampleCount = totalAudioPackets.length;
+		const timebase = probe.audioTimebase;
+		if (!timebase) {
+			console.warn("No timebase found for audio stream");
+			continue;
 		}
+		const timescale = Math.round(timebase.den / timebase.num);
 		let trackStartTimeOffsetMs;
-		if (allPackets.length > 0) {
-			const firstPacketTime = allPackets[0].timestamp;
-			if (Math.abs(firstPacketTime) > .01) trackStartTimeOffsetMs = firstPacketTime * 1e3;
+		const allAudioPackets = probe.packets.filter((p) => p.stream_index === audioStream.index);
+		if (allAudioPackets.length > 0) {
+			const presentationTime = allAudioPackets[0].pts_time;
+			if (Math.abs(presentationTime) > .01) trackStartTimeOffsetMs = presentationTime * 1e3;
 		}
 		if (startTimeOffsetMs !== void 0) trackStartTimeOffsetMs = startTimeOffsetMs;
-		const timescale = Math.round(track.timeResolution);
-		let accumulatedDts = 0;
-		let accumulatedCts = 0;
-		for (const timing of audioFragmentTimings) {
-			const fragment = mediaFragments[timing.fragmentIndex];
+		log(`Processing ${fragmentTimingData.length} fragments for audio stream ${audioStream.index}`);
+		for (const fragmentData of fragmentTimingData) {
+			const fragment = mediaFragments[fragmentData.fragmentIndex];
+			const audioPackets = fragmentData.audioPackets;
+			log(`Fragment ${fragmentData.fragmentIndex}: ${audioPackets.length} audio packets`);
+			if (audioPackets.length === 0) {
+				log(`Skipping fragment ${fragmentData.fragmentIndex} - no audio packets`);
+				continue;
+			}
+			const firstPacket = audioPackets[0];
+			const segmentCts = Math.round(firstPacket.pts * timescale / timebase.den);
+			const segmentDts = Math.round(firstPacket.dts * timescale / timebase.den);
+			const nextFragmentData = fragmentTimingData[fragmentData.fragmentIndex + 1];
+			const nextFirstPacket = nextFragmentData?.audioPackets[0];
+			let segmentDuration;
+			if (nextFirstPacket) {
+				const nextSegmentCts = Math.round(nextFirstPacket.pts * timescale / timebase.den);
+				segmentDuration = nextSegmentCts - segmentCts;
+			} else {
+				const allAudioPackets$1 = probe.packets.filter((p) => {
+					const stream = audioStreams.find((s) => s.index === p.stream_index);
+					return stream?.codec_type === "audio";
+				}).sort((a, b) => a.pts - b.pts);
+				const lastPacket = allAudioPackets$1[allAudioPackets$1.length - 1];
+				const streamEnd = Math.round((lastPacket.pts + (lastPacket.duration || 0)) * timescale / timebase.den);
+				segmentDuration = streamEnd - segmentCts;
+			}
 			segments.push({
-				cts: accumulatedCts,
-				dts: accumulatedDts,
-				duration: timing.duration,
+				cts: segmentCts,
+				dts: segmentDts,
+				duration: segmentDuration,
 				offset: fragment.offset,
 				size: fragment.size
 			});
-			accumulatedDts += timing.duration;
-			accumulatedCts += timing.duration;
-			totalDuration += timing.duration / timescale;
 		}
-		const finalTrackId = trackIdMapping?.[track.id] ?? track.id;
+		const totalDuration = segments.reduce((sum, seg) => sum + seg.duration, 0);
+		const finalTrackId = trackIdMapping?.[audioStream.index] ?? audioStream.index + 1;
 		trackIndexes[finalTrackId] = {
 			track: finalTrackId,
 			type: "audio",
-			channel_count: track.numberOfChannels,
-			sample_rate: track.sampleRate,
-			sample_size: 16,
-			sample_count: sampleCount,
-			timescale: Math.round(track.timeResolution),
-			codec: await track.getCodecParameterString() || "",
-			duration: Math.round(totalDuration * track.timeResolution),
+			channel_count: audioStream.channels,
+			sample_rate: Number(audioStream.sample_rate),
+			sample_size: audioStream.bits_per_sample,
+			sample_count: totalSampleCount,
+			timescale,
+			codec: audioStream.codec_tag_string || audioStream.codec_name || "",
+			duration: totalDuration,
 			startTimeOffsetMs: trackStartTimeOffsetMs,
 			initSegment: {
 				offset: 0,

package/dist/tasks/generateTrackFragmentIndex.js

@@ -25,7 +25,7 @@ const generateTrackFragmentIndexFromPath = async (absolutePath) => {
 		const trackId = streamIndex + 1;
 		log(`Processing track ${trackId} (${stream.codec_type})`);
 		const trackStream = probe.createTrackReadstream(streamIndex);
-		const trackIdMapping = { 1: trackId };
+		const trackIdMapping = { 0: trackId };
 		const singleTrackIndexes = await generateFragmentIndex(trackStream, startTimeOffsetMs, trackIdMapping);
 		Object.assign(trackFragmentIndexes, singleTrackIndexes);
 	}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@editframe/assets",
-  "version": "0.18.21-beta.0",
+  "version": "0.18.22-beta.0",
   "description": "",
   "exports": {
     ".": {

package/src/tasks/generateTrackFragmentIndex.test.ts

@@ -46,10 +46,8 @@ describe("generateTrackFragmentIndex", () => {
         assert.isNumber(track.sample_rate, `Audio track ${trackId} should have sample_rate`);
         assert.isNumber(track.sample_size, `Audio track ${trackId} should have sample_size`);
       }
-
-      console.log(`Track ${trackId} (${track.type}): ${track.segments.length} segments, ${track.sample_count} samples`);
     }
-  }, 20000);
+  });
 
   test("should handle single track files", async () => {
     const fragmentIndex = await generateTrackFragmentIndexFromPath("test-assets/frame-count.mp4");
@@ -60,9 +58,7 @@ describe("generateTrackFragmentIndex", () => {
     const track = fragmentIndex[trackIds[0]!]!;
     assert.equal(track.type, "video", "Should be video track");
     assert.isAbove(track.segments.length, 0, "Should have segments");
-
-    console.log(`Single track: ${track.segments.length} segments, ${track.sample_count} samples`);
-  }, 15000);
+  });
 
   test("should generate consistent results with original implementation", async () => {
     // Test that the new implementation produces similar structure to the old one
@@ -87,8 +83,6 @@ describe("generateTrackFragmentIndex", () => {
     assert.isAbove(audioTrack.channel_count, 0, "Audio should have channels");
     assert.isAbove(audioTrack.sample_rate, 0, "Audio should have sample rate");
     assert.isAbove(audioTrack.segments.length, 0, "Audio should have segments");
-
-    console.log(`Consistent results: video ${videoTrack.segments.length} segments, audio ${audioTrack.segments.length} segments`);
   }, 20000);
 
   test("should preserve timing offset detection", async () => {
@@ -98,12 +92,8 @@ describe("generateTrackFragmentIndex", () => {
     const trackIds = Object.keys(fragmentIndex).map(Number);
     const track = fragmentIndex[trackIds[0]!]!;
 
-    if (track.type === 'video' && track.startTimeOffsetMs !== undefined) {
-      assert.isNumber(track.startTimeOffsetMs, "Should have timing offset");
-      console.log(`Detected timing offset: ${track.startTimeOffsetMs}ms`);
-    } else {
-      console.log("No timing offset detected (expected for this file)");
-    }
+    assert.equal(track.startTimeOffsetMs, 200);
+    assert.equal(track.type, "video");
 
     // Should still have valid timing data
     assert.isAbove(track.duration, 0, "Should have positive duration");

package/src/tasks/generateTrackFragmentIndex.ts

@@ -49,7 +49,7 @@ export const generateTrackFragmentIndexFromPath = async (
 
     // Generate single-track file and its fragment index
     const trackStream = probe.createTrackReadstream(streamIndex);
-    const trackIdMapping = { 1: trackId }; // Map single-track ID 1 to original track ID
+    const trackIdMapping = { 0: trackId }; // Map single-track stream index 0 to original track ID
 
     const singleTrackIndexes = await generateFragmentIndex(
       trackStream,