agentroam-runtime-darwin-arm64 0.2.0-preview.68 → 0.2.0-preview.69

package/runtime/node_modules/mediabunny/dist/modules/src/isobmff/isobmff-muxer.js

@@ -13,14 +13,15 @@ import { assert, computeRationalApproximation, last, promiseWithResolvers, simpl
 import { MovOutputFormat, CmafOutputFormat } from '../output-format.js';
 import { inlineTimestampRegex } from '../subtitles.js';
 import { aacChannelMap, aacFrequencyTable, buildAacAudioSpecificConfig } from '../../shared/aac-misc.js';
-import { parsePcmCodec, PCM_AUDIO_CODECS, validateAudioChunkMetadata, validateSubtitleMetadata, validateVideoChunkMetadata, } from '../codec.js';
+import { OPUS_PRE_ROLL_DURATION, OPUS_SAMPLE_RATE, parsePcmCodec, PCM_AUDIO_CODECS, validateAudioChunkMetadata, validateSubtitleMetadata, validateVideoChunkMetadata, } from '../codec.js';
 import { MAX_ADTS_FRAME_HEADER_SIZE, MIN_ADTS_FRAME_HEADER_SIZE, readAdtsFrameHeader } from '../adts/adts-reader.js';
 import { FileSlice } from '../reader.js';
-import { concatNalUnitsInLengthPrefixed, extractAvcDecoderConfigurationRecord, extractHevcDecoderConfigurationRecord, iterateNalUnitsInAnnexB, serializeAvcDecoderConfigurationRecord, serializeHevcDecoderConfigurationRecord, } from '../codec-data.js';
+import { concatNalUnitsInLengthPrefixed, extractAvcDecoderConfigurationRecord, extractHevcDecoderConfigurationRecord, iterateNalUnitsInAnnexB, parseOpusTocByte, serializeAvcDecoderConfigurationRecord, serializeHevcDecoderConfigurationRecord, } from '../codec-data.js';
 import { buildIsobmffMimeType } from './isobmff-misc.js';
 import { MAX_BOX_HEADER_SIZE, MIN_BOX_HEADER_SIZE } from './isobmff-reader.js';
 export const GLOBAL_TIMESCALE = 57600; // LCM of a bunch of common frame rates (24, 25, 30, 60, 144, ...)
 const TIMESTAMP_OFFSET = 2_082_844_800; // Seconds between Jan 1 1904 and Jan 1 1970
+const MIN_CHUNK_DURATION = 0.5; // In seconds
 export const getTrackMetadata = (trackData) => {
     const metadata = {};
     const track = trackData.track;
@@ -339,6 +340,9 @@ export class IsobmffMuxer extends Muxer {
             if (track.source._codec === 'dts') {
                 throw new Error('DTS requires a priming packet.');
             }
+            if (track.source._codec === 'truehd') {
+                throw new Error('TrueHD requires a priming packet.');
+            }
         }
         const newTrackData = {
             muxer: this,
@@ -353,6 +357,11 @@ export class IsobmffMuxer extends Muxer {
                 expectedNextPcmPacketTimestamp: null,
                 requiresAdtsStripping,
                 primingPacket: packet,
+                preroll: {
+                    packetCount: 0,
+                    accumulatedAudioSamples: 0,
+                    audioSampleDurationHistory: [],
+                },
             },
             timescale: decoderConfig.sampleRate,
             samples: [],
@@ -437,7 +446,7 @@ export class IsobmffMuxer extends Muxer {
                 packetData = concatNalUnitsInLengthPrefixed(nalUnits, 4);
             }
             this.validateTimestamp(trackData.track, packet.timestamp, packet.type === 'key');
-            const internalSample = this.createSampleForTrack(trackData, packetData, packet.timestamp, packet.duration, packet.type);
+            const internalSample = this.createSampleForTrack(trackData, packetData, packet.timestamp, packet.duration, packet.type, 0);
             await this.registerSample(trackData, internalSample);
         }
         finally {
@@ -481,7 +490,32 @@ export class IsobmffMuxer extends Muxer {
                 }
                 trackData.info.expectedNextPcmPacketTimestamp = timestamp + duration;
             }
-            const internalSample = this.createSampleForTrack(trackData, packetData, timestamp, duration, packet.type);
+            const { preroll } = trackData.info;
+            // Let's specify how many samples (packets) worth of preroll are needed to decode audio properly, based on
+            // the codec.
+            let prerollSamples = 0;
+            if (track.source._codec === 'aac' || track.source._codec === 'vorbis') {
+                prerollSamples = -Math.min(1, preroll.packetCount);
+            }
+            else if (track.source._codec === 'mp3') {
+                // One packet of overlap plus one for the bit reservoir, which usually doesn't reach back further.
+                // ESTIMATE!!
+                prerollSamples = -Math.min(2, preroll.packetCount);
+            }
+            else if (track.source._codec === 'opus') {
+                const OPUS_PREROLL_SAMPLES = Math.round(OPUS_PRE_ROLL_DURATION * OPUS_SAMPLE_RATE);
+                prerollSamples = -preroll.audioSampleDurationHistory.length;
+                const toc = parseOpusTocByte(packetData);
+                preroll.accumulatedAudioSamples += toc.durationInSamples;
+                preroll.audioSampleDurationHistory.push(toc.durationInSamples);
+                while (preroll.accumulatedAudioSamples - preroll.audioSampleDurationHistory[0]
+                    >= OPUS_PREROLL_SAMPLES) {
+                    preroll.accumulatedAudioSamples -= preroll.audioSampleDurationHistory.shift();
+                    assert(preroll.audioSampleDurationHistory.length > 0);
+                }
+            }
+            preroll.packetCount++;
+            const internalSample = this.createSampleForTrack(trackData, packetData, timestamp, duration, packet.type, prerollSamples);
             await this.registerSample(trackData, internalSample);
         }
         finally {
@@ -495,7 +529,7 @@ export class IsobmffMuxer extends Muxer {
             const { sampleSize, silentValue } = parsePcmCodec(trackData.info.decoderConfig.codec);
             const samplesNeeded = deltaInTimescale * trackData.info.numberOfChannels;
             const data = new Uint8Array(sampleSize * samplesNeeded).fill(silentValue);
-            const paddingSample = this.createSampleForTrack(trackData, new Uint8Array(data.buffer), timestamp, duration, 'key');
+            const paddingSample = this.createSampleForTrack(trackData, new Uint8Array(data.buffer), timestamp, duration, 'key', 0);
             await this.registerSample(trackData, paddingSample);
         }
         return duration;
@@ -542,7 +576,7 @@ export class IsobmffMuxer extends Muxer {
                 const box = vtte();
                 this.auxBoxWriter.writeBox(box);
                 const body = this.auxTarget._getSlice(0, this.auxWriter.getPos());
-                const sample = this.createSampleForTrack(trackData, body, trackData.lastCueEndTimestamp, sampleStart - trackData.lastCueEndTimestamp, 'key');
+                const sample = this.createSampleForTrack(trackData, body, trackData.lastCueEndTimestamp, sampleStart - trackData.lastCueEndTimestamp, 'key', 0);
                 await this.registerSample(trackData, sample);
                 trackData.lastCueEndTimestamp = sampleStart;
             }
@@ -575,12 +609,12 @@ export class IsobmffMuxer extends Muxer {
                 }
             }
             const body = this.auxTarget._getSlice(0, this.auxWriter.getPos());
-            const sample = this.createSampleForTrack(trackData, body, sampleStart, sampleEnd - sampleStart, 'key');
+            const sample = this.createSampleForTrack(trackData, body, sampleStart, sampleEnd - sampleStart, 'key', 0);
             await this.registerSample(trackData, sample);
             trackData.lastCueEndTimestamp = sampleEnd;
         }
     }
-    createSampleForTrack(trackData, data, timestamp, duration, type) {
+    createSampleForTrack(trackData, data, timestamp, duration, type, prerollSamples) {
         const sample = {
             timestamp,
             decodeTimestamp: timestamp, // This may be refined later
@@ -589,6 +623,7 @@ export class IsobmffMuxer extends Muxer {
             size: data.byteLength,
             type,
             timescaleUnitsToNextSample: intoTimescale(duration, trackData.timescale), // Will be refined
+            prerollSamples,
         };
         return sample;
     }
@@ -727,16 +762,8 @@ export class IsobmffMuxer extends Muxer {
             this.processTimestamps(trackData, sample);
         }
         trackData.timestampProcessingQueue.push(sample);
-        if (this.isFragmented) {
-            trackData.sampleQueue.push(sample);
-            await this.interleaveSamples();
-        }
-        else if (this.fastStart === 'reserve') {
-            await this.registerSampleFastStartReserve(trackData, sample);
-        }
-        else {
-            await this.addSampleToTrack(trackData, sample);
-        }
+        trackData.sampleQueue.push(sample);
+        await this.interleaveSamples();
     }
     async addSampleToTrack(trackData, sample) {
         if (!this.isFragmented) {
@@ -780,12 +807,21 @@ export class IsobmffMuxer extends Muxer {
                 }
             }
             else {
-                beginNewChunk = currentChunkDuration >= 0.5; // Chunk is long enough, we need a new one
+                beginNewChunk = currentChunkDuration >= MIN_CHUNK_DURATION; // Chunk is long enough, we need a new one
             }
         }
         if (beginNewChunk) {
-            if (trackData.currentChunk) {
-                await this.finalizeCurrentChunk(trackData);
+            if (!this.isFragmented) {
+                // See if we can finalize the chunk of the other tracks too. We can do this because samples are added
+                // interleaved by timestamp anyway. It's important that chunks are finalized and emitted in
+                // chronological order because some players behave better if this is the case.
+                const tracksToFinalize = this.trackDatas
+                    .filter(x => (x.currentChunk
+                    && sample.timestamp - x.currentChunk.startTimestamp >= MIN_CHUNK_DURATION))
+                    .sort((a, b) => a.currentChunk.startTimestamp - b.currentChunk.startTimestamp);
+                for (const otherTrackData of tracksToFinalize) {
+                    await this.finalizeCurrentChunk(otherTrackData);
+                }
             }
             trackData.currentChunk = {
                 startTimestamp: sample.timestamp,
@@ -806,13 +842,14 @@ export class IsobmffMuxer extends Muxer {
     async finalizeCurrentChunk(trackData) {
         assert(!this.isFragmented);
         assert(this.writer);
-        if (!trackData.currentChunk)
-            return;
-        trackData.finalizedChunks.push(trackData.currentChunk);
-        this.finalizedChunks.push(trackData.currentChunk);
-        let sampleCount = trackData.currentChunk.samples.length;
+        const chunk = trackData.currentChunk;
+        assert(chunk);
+        trackData.currentChunk = null;
+        trackData.finalizedChunks.push(chunk);
+        this.finalizedChunks.push(chunk);
+        let sampleCount = chunk.samples.length;
         if (trackData.type === 'audio' && trackData.info.requiresPcmTransformation) {
-            sampleCount = trackData.currentChunk.samples
+            sampleCount = chunk.samples
                 .reduce((acc, sample) => acc + intoTimescale(sample.duration, trackData.timescale), 0);
         }
         if (trackData.compactlyCodedChunkTable.length === 0
@@ -823,12 +860,12 @@ export class IsobmffMuxer extends Muxer {
             });
         }
         if (this.fastStart === 'in-memory') {
-            trackData.currentChunk.offset = 0; // We'll compute the proper offset when finalizing
+            chunk.offset = 0; // We'll compute the proper offset when finalizing
             return;
         }
         // Write out the data
-        trackData.currentChunk.offset = this.writer.getPos();
-        for (const sample of trackData.currentChunk.samples) {
+        chunk.offset = this.writer.getPos();
+        for (const sample of chunk.samples) {
             assert(sample.data);
             this.writer.write(sample.data);
             sample.data = null; // Can be GC'd
@@ -836,10 +873,12 @@ export class IsobmffMuxer extends Muxer {
         await this.writer.flush();
     }
     async interleaveSamples(isFinalCall = false) {
-        assert(this.isFragmented);
         if (!isFinalCall && !this.allTracksAreKnown()) {
             return; // We can't interleave yet as we don't yet know how many tracks we'll truly have
         }
+        if (this.fastStart === 'reserve' && !this.mdat) {
+            await this.createFastStartReserveMdat();
+        }
         outer: while (true) {
             let trackWithMinTimestamp = null;
             let minTimestamp = Infinity;
@@ -969,18 +1008,6 @@ export class IsobmffMuxer extends Muxer {
             await this.writer.flush();
         }
     }
-    async registerSampleFastStartReserve(trackData, sample) {
-        if (this.allTracksAreKnown()) {
-            if (!this.mdat) {
-                await this.createFastStartReserveMdat();
-            }
-            await this.addSampleToTrack(trackData, sample);
-        }
-        else {
-            // Queue it for when we know all tracks
-            trackData.sampleQueue.push(sample);
-        }
-    }
     async createFastStartReserveMdat() {
         assert(this.writer);
         assert(this.boxWriter);
@@ -998,13 +1025,6 @@ export class IsobmffMuxer extends Muxer {
         }
         this.mdat = mdat(true);
         this.boxWriter.writeBox(this.mdat);
-        // Now write everything that was queued
-        for (const trackData of this.trackDatas) {
-            for (const sample of trackData.sampleQueue) {
-                await this.addSampleToTrack(trackData, sample);
-            }
-            trackData.sampleQueue.length = 0;
-        }
     }
     computeSampleTableSizeUpperBound() {
         assert(this.fastStart === 'reserve');
@@ -1043,10 +1063,8 @@ export class IsobmffMuxer extends Muxer {
         if (this.allTracksAreKnown()) {
             this.allTracksKnown.resolve();
         }
-        if (this.isFragmented) {
-            // Since a track is now closed, we may be able to write out chunks that were previously waiting
-            await this.interleaveSamples();
-        }
+        // Since a track is now closed, we may be able to write out chunks that were previously waiting
+        await this.interleaveSamples();
         release();
     }
     ensureOneEnabledTrack() {
@@ -1091,9 +1109,6 @@ export class IsobmffMuxer extends Muxer {
         const release = await this.mutex.acquire();
         this.allTracksKnown.resolve();
         this.ensureOneEnabledTrack();
-        if (!this.mdat && this.fastStart === 'reserve') {
-            await this.createFastStartReserveMdat();
-        }
         for (const trackData of this.trackDatas) {
             trackData.closed = true;
             if (trackData.type === 'subtitle' && trackData.track.source._codec === 'webvtt') {
@@ -1101,13 +1116,18 @@ export class IsobmffMuxer extends Muxer {
             }
             this.processTimestamps(trackData);
         }
+        await this.interleaveSamples(true);
         if (this.isFragmented) {
-            await this.interleaveSamples(true);
             await this.finalizeFragment(false); // Don't flush the last fragment as we will flush it with the mfra box
         }
         else {
-            for (const trackData of this.trackDatas) {
+            const tracksToFinalize = this.trackDatas
+                .filter(x => x.currentChunk)
+                .sort((a, b) => a.currentChunk.startTimestamp - b.currentChunk.startTimestamp);
+            for (const trackData of tracksToFinalize) {
                 await this.finalizeCurrentChunk(trackData);
+            }
+            for (const trackData of this.trackDatas) {
                 // Now that we have all samples, we can replace the nominal bitrates with measured ones
                 const span = presentationSpan(trackData);
                 if (span > 0) {

package/runtime/node_modules/mediabunny/dist/modules/src/matroska/ebml.js

@@ -101,6 +101,7 @@ export var EBMLId;
     EBMLId[EBMLId["CueTrackPositions"] = 183] = "CueTrackPositions";
     EBMLId[EBMLId["CueTrack"] = 247] = "CueTrack";
     EBMLId[EBMLId["CueClusterPosition"] = 241] = "CueClusterPosition";
+    EBMLId[EBMLId["CueRelativePosition"] = 240] = "CueRelativePosition";
     EBMLId[EBMLId["Colour"] = 21936] = "Colour";
     EBMLId[EBMLId["MatrixCoefficients"] = 21937] = "MatrixCoefficients";
     EBMLId[EBMLId["TransferCharacteristics"] = 21946] = "TransferCharacteristics";
@@ -669,6 +670,7 @@ export const CODEC_STRING_MAP = {
     'ac3': 'A_AC3',
     'eac3': 'A_EAC3',
     'dts': 'A_DTS',
+    'truehd': 'A_TRUEHD',
     'pcm-u8': 'A_PCM/INT/LIT',
     'pcm-s16': 'A_PCM/INT/LIT',
     'pcm-s16be': 'A_PCM/INT/BIG',

package/runtime/node_modules/mediabunny/dist/modules/src/matroska/matroska-demuxer.js

@@ -6,7 +6,7 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 import { extractAv1CodecInfoFromPacket, extractDtsFourCcFromPacket, extractAvcDecoderConfigurationRecord, extractHevcDecoderConfigurationRecord, extractProresCodecInfoFromPacket, extractVp9CodecInfoFromPacket, } from '../codec-data.js';
-import { extractAudioCodecString, extractColorSpace, extractVideoCodecString, OPUS_SAMPLE_RATE, PRORES_FOURCCS, } from '../codec.js';
+import { AUDIO_CODECS_WITH_DELTA_PACKETS, extractAudioCodecString, extractColorSpace, extractVideoCodecString, OPUS_SAMPLE_RATE, PRORES_FOURCCS, } from '../codec.js';
 import { Demuxer } from '../demuxer.js';
 import { Logging } from '../logging.js';
 import { AttachedFile, DEFAULT_TRACK_DISPOSITION } from '../metadata.js';
@@ -324,11 +324,12 @@ export class MatroskaDemuxer extends Demuxer {
             this.currentSegment.timestampScale = 1e6;
             this.currentSegment.timestampFactor = 1e9 / 1e6;
         }
-        // Compute default duration for all tracks now that we have the timestamp factor
+        // Compute default duration and codec delay for all tracks now that we have the timestamp factor
         for (const track of this.currentSegment.tracks) {
             if (track.defaultDurationNs !== null) {
                 track.defaultDuration = (this.currentSegment.timestampFactor * track.defaultDurationNs) / 1e9;
             }
+            track.codecDelay = Math.round((this.currentSegment.timestampFactor * track.codecDelayNs) / 1e9);
         }
         // Assign cue points to their respective tracks
         for (const cuePoint of this.currentSegment.cuePoints) {
@@ -772,6 +773,8 @@ export class MatroskaDemuxer extends Demuxer {
                         codecPrivate: null,
                         defaultDuration: null,
                         defaultDurationNs: null,
+                        codecDelay: 0,
+                        codecDelayNs: 0,
                         name: null,
                         languageCode: 'eng', // The default in Matroska
                         hasLanguageBcp47: false,
@@ -899,6 +902,9 @@ export class MatroskaDemuxer extends Demuxer {
                                     this.currentTrack.info.dtsFormat = 'dtsl';
                                 }
                             }
+                            else if (codecIdWithoutSuffix === CODEC_STRING_MAP.truehd) {
+                                this.currentTrack.info.codec = 'truehd';
+                            }
                             else if (this.currentTrack.codecId === 'A_PCM/INT/LIT') {
                                 if (this.currentTrack.info.bitDepth === 8) {
                                     this.currentTrack.info.codec = 'pcm-u8';
@@ -1076,6 +1082,14 @@ export class MatroskaDemuxer extends Demuxer {
                 }
                 ;
                 break;
+            case EBMLId.CodecDelay:
+                {
+                    if (!this.currentTrack)
+                        break;
+                    this.currentTrack.codecDelayNs = readUnsignedInt(slice, size);
+                }
+                ;
+                break;
             case EBMLId.Name:
                 {
                     if (!this.currentTrack)
@@ -1389,10 +1403,12 @@ export class MatroskaDemuxer extends Demuxer {
                     const flags = readU8(blockSlice);
                     const lacing = (flags >> 1) & 0x3; // If the block is laced, we'll expand it later
                     let isKeyFrame = !!(flags & 0x80);
-                    if (trackData.track.info?.type === 'audio' && trackData.track.info.codec) {
+                    if (trackData.track.info?.type === 'audio'
+                        && trackData.track.info.codec
+                        && !AUDIO_CODECS_WITH_DELTA_PACKETS.includes(trackData.track.info.codec)) {
                         // Some files don't mark their audio packets as key packets (I'm looking at you, Firefox). But, we
-                        // can fix this in most cases: if we recognize the codec of the track, then we know every packet is
-                        // necessarily a key packet, no matter what the container says.
+                        // can fix this in most cases: if we recognize the codec of the track and it has no delta packets,
+                        // then we know every packet is necessarily a key packet, no matter what the container says.
                         // https://github.com/Vanilagy/mediabunny/issues/192
                         isKeyFrame = true;
                     }
@@ -1897,9 +1913,10 @@ export class MatroskaDemuxer extends Demuxer {
         if (segment.duration <= 0) {
             return null;
         }
+        const minCodecDelay = Math.min(...segment.tracks.map(x => x.codecDelay));
         // We treat the Duration field as the end timestamp of the segment, since this is what FFmpeg treats it as.
         // The spec unfortunately does not clarify what it means by duration here.
-        return segment.duration / segment.timestampFactor;
+        return (segment.duration - minCodecDelay) / segment.timestampFactor;
     }
 }
 class MatroskaTrackBacking {
@@ -1985,7 +2002,8 @@ class MatroskaTrackBacking {
         // Do a little rounding to catch cases where the result is very close to an integer. If it is, it's likely
         // that the number was originally an integer divided by the timescale. For stability, it's best
         // to return the integer in this case.
-        return roundIfAlmostInteger(timestamp * this.internalTrack.segment.timestampFactor);
+        return roundIfAlmostInteger(timestamp * this.internalTrack.segment.timestampFactor)
+            + this.internalTrack.codecDelay;
     }
     async getPacket(timestamp, options) {
         const timestampInTimescale = this.intoTimescale(timestamp);
@@ -2121,8 +2139,9 @@ class MatroskaTrackBacking {
             block.postProcessed = true;
         }
         const data = options.metadataOnly ? PLACEHOLDER_DATA : block.data;
-        const timestamp = block.timestamp / this.internalTrack.segment.timestampFactor;
-        const duration = block.duration / this.internalTrack.segment.timestampFactor;
+        const { codecDelay, segment } = this.internalTrack;
+        const timestamp = (block.timestamp - codecDelay) / segment.timestampFactor;
+        const duration = block.duration / segment.timestampFactor;
         const sideData = {};
         if (block.mainAdditional && this.internalTrack.info?.type === 'video' && this.internalTrack.info.alphaMode) {
             sideData.alpha = options.metadataOnly ? PLACEHOLDER_DATA : block.mainAdditional;