mediasoup 3.24.2 → 3.26.0

package/worker/include/RTC/SimulcastProducerStreamManager.hpp

@@ -80,6 +80,9 @@ namespace RTC
 		int16_t spatialLayerToSync{ -1 };
 		// Timestamp synchronization.
 		int16_t tsReferenceSpatialLayer{ -1 };
+		// Spatial layer that was the RTP timestamp reference the last time its capture
+		// instant was known upon a received Sender Report.
+		int16_t tsReferenceSpatialLayerWithCaptureMapping{ -1 };
 		uint32_t tsOffset{ 0u };
 		bool keyFrameForTsOffsetRequested{ false };
 		// Old-packet filtering after spatial switch.

package/worker/include/RTC/Transport.hpp

@@ -17,6 +17,7 @@
 #include "RTC/RTP/HeaderExtensionIds.hpp"
 #include "RTC/RTP/Packet.hpp"
 #include "RTC/RateCalculator.hpp"
+#include "RTC/RemoteCaptureTimeEstimator.hpp"
 #include "RTC/RtpListener.hpp"
 #include "RTC/SCTP/public/AssociationInterface.hpp"
 #include "RTC/SCTP/public/AssociationListenerInterface.hpp"
@@ -83,11 +84,12 @@ namespace RTC
 			  bool first) = 0;
 			virtual void OnTransportProducerRtpPacketReceived(
 			  RTC::Transport* transport, RTC::Producer* producer, RTC::RTP::Packet* packet) = 0;
-			virtual void OnTransportNeedWorstRemoteFractionLost(
-			  RTC::Transport* transport,
-			  RTC::Producer* producer,
-			  uint32_t mappedSsrc,
-			  uint8_t& worstRemoteFractionLost) = 0;
+			/**
+			 * Obtains the worst remote fraction lost of all the Consumers consuming the
+			 * given mapped SSRC of the given Producer.
+			 */
+			virtual uint8_t OnTransportNeedWorstRemoteFractionLost(
+			  RTC::Transport* transport, RTC::Producer* producer, uint32_t mappedSsrc) = 0;
 			virtual void OnTransportNewConsumer(
 			  RTC::Transport* transport, RTC::Consumer* consumer, const std::string& producerId) = 0;
 			virtual void OnTransportConsumerClosed(RTC::Transport* transport, RTC::Consumer* consumer) = 0;
@@ -264,8 +266,10 @@ namespace RTC
 		  RTC::Producer* producer, RTC::RTP::RtpStreamRecv* rtpStream, bool first) override;
 		void OnProducerRtpPacketReceived(RTC::Producer* producer, RTC::RTP::Packet* packet) override;
 		void OnProducerSendRtcpPacket(RTC::Producer* producer, RTC::RTCP::Packet* packet) override;
-		void OnProducerNeedWorstRemoteFractionLost(
-		  RTC::Producer* producer, uint32_t mappedSsrc, uint8_t& worstRemoteFractionLost) override;
+		uint8_t OnProducerNeedWorstRemoteFractionLost(RTC::Producer* producer, uint32_t mappedSsrc) override;
+		std::optional<uint64_t> OnProducerNeedLocalCaptureMs(
+		  RTC::Producer* producer, const RTC::RTP::RtpStreamRecv* rtpStream, uint32_t ts) override;
+		std::optional<int64_t> OnProducerNeedRemoteClockOffsetMs(const RTC::Producer* producer) override;
 
 		/* Pure virtual methods inherited from RTC::Consumer::Listener. */
 	public:
@@ -367,6 +371,10 @@ namespace RTC
 		ankerl::unordered_dense::map<uint16_t, RTC::DataConsumer*> mapSctpStreamIdDataConsumers;
 		ankerl::unordered_dense::map<uint32_t, RTC::Consumer*> mapSsrcConsumer;
 		ankerl::unordered_dense::map<uint32_t, RTC::Consumer*> mapRtxSsrcConsumer;
+		// Capture time estimator of each sender publishing to this Transport, keyed by
+		// its CNAME.
+		ankerl::unordered_dense::map<std::string, RTC::RemoteCaptureTimeEstimator>
+		  mapCnameRemoteCaptureTimeEstimator;
 		TimerHandleInterface* rtcpTimer{ nullptr };
 		// Allocated by this.
 		std::unique_ptr<RTC::SCTP::AssociationInterface> sctpAssociation{ nullptr };

package/worker/include/Shared.hpp

@@ -56,6 +56,11 @@ public:
 		return DepLibUV::GetTimeUsInt64();
 	}
 
+	uint64_t GetNtpOffsetMs() override
+	{
+		return DepLibUV::GetNtpOffsetMs();
+	}
+
 private:
 	std::unique_ptr<Channel::ChannelMessageRegistrator> channelMessageRegistrator;
 	std::unique_ptr<Channel::ChannelNotifier> channelNotifier;

package/worker/include/SharedInterface.hpp

@@ -72,6 +72,12 @@ public:
 	 * @todo Remove once not needed.
 	 */
 	virtual int64_t GetTimeUsInt64() = 0;
+
+	/**
+	 * Distance from the clock above to the NTP epoch (ms), which is what has to be added
+	 * to it to obtain the NTP timestamps of the RTCP we generate.
+	 */
+	virtual uint64_t GetNtpOffsetMs() = 0;
 };
 
 #endif

package/worker/include/Utils.hpp

@@ -500,9 +500,11 @@ namespace Utils
 
 	class Time
 	{
-	private:
+	public:
 		// Seconds from Jan 1, 1900 to Jan 1, 1970.
-		static constexpr uint32_t UnixNtpOffset{ 0x83AA7E80 };
+		static constexpr uint32_t UnixNtpOffsetSec{ 0x83AA7E80 };
+
+	private:
 		// NTP fractional unit.
 		static constexpr uint64_t NtpFractionalUnit{ 1LL << 32 };
 
@@ -517,7 +519,11 @@ namespace Utils
 		{
 			Time::Ntp ntp{}; // NOLINT(cppcoreguidelines-pro-type-member-init)
 
-			ntp.seconds = ms / 1000;
+			// NOTE: The truncation is intended. Seconds are 32 bits wide in NTP, so they wrap
+			// every 2^32 seconds since Jan 1, 1900, being Feb 7, 2036 the next time it
+			// happens. That is the NTP era and receivers deal with it by doing modular
+			// arithmetic, so there is nothing to protect against here.
+			ntp.seconds = static_cast<uint32_t>(ms / 1000);
 			ntp.fractions =
 			  static_cast<uint32_t>((static_cast<double>(ms % 1000) / 1000) * NtpFractionalUnit);
 
@@ -536,6 +542,41 @@ namespace Utils
 		{
 			return static_cast<uint32_t>(((ms << 18) + 500) / 1000) & 0x00FFFFFF;
 		}
+
+		/**
+		 * Convert milliseconds into signed Q32.32 fixed point seconds, which is how the
+		 * `abs-capture-time` RTP header extension encodes its clock offset, being the
+		 * capture timestamp itself unsigned instead (UQ32.32).
+		 *
+		 * @returns No value if the given instant does not fit in the format, whose seconds
+		 * are 32 bits wide.
+		 *
+		 * @see https://datatracker.ietf.org/doc/html/draft-ietf-avtcore-abs-capture-time-00
+		 */
+		static std::optional<int64_t> TimeMs2Q32x32(int64_t ms)
+		{
+			// The seconds of the format are 32 bits wide, so from here on it does not fit.
+			static constexpr int64_t OutOfRangeMs{ (1LL << 31) * 1000 };
+
+			if (ms >= OutOfRangeMs || ms <= -OutOfRangeMs)
+			{
+				return std::nullopt;
+			}
+
+			return static_cast<int64_t>(
+			  std::round(static_cast<double>(ms) * (static_cast<double>(NtpFractionalUnit) / 1000)));
+		}
+
+		/**
+		 * Convert signed Q32.32 fixed point seconds into milliseconds.
+		 *
+		 * @see https://datatracker.ietf.org/doc/html/draft-ietf-avtcore-abs-capture-time-00
+		 */
+		static int64_t Q32x32ToTimeMs(int64_t q32x32)
+		{
+			return static_cast<int64_t>(
+			  std::round(static_cast<double>(q32x32) * (1000 / static_cast<double>(NtpFractionalUnit))));
+		}
 	};
 
 	class BitStream

package/worker/meson.build

@@ -124,6 +124,8 @@ common_sources = [
   'src/RTC/PortManager.cpp',
   'src/RTC/Producer.cpp',
   'src/RTC/RateCalculator.cpp',
+  'src/RTC/RemoteCaptureTimeEstimator.cpp',
+  'src/RTC/RemoteClockOffsetEstimator.cpp',
   'src/RTC/Router.cpp',
   'src/RTC/RtcLogger.cpp',
   'src/RTC/RtpListener.cpp',
@@ -429,6 +431,8 @@ test_sources = [
   'test/src/RTC/TestNackGenerator.cpp',
   'test/src/RTC/TestPortManager.cpp',
   'test/src/RTC/TestRateCalculator.cpp',
+  'test/src/RTC/TestRemoteCaptureTimeEstimator.cpp',
+  'test/src/RTC/TestRemoteClockOffsetEstimator.cpp',
   'test/src/RTC/TestRtpEncodingParameters.cpp',
   'test/src/RTC/TestSeqManager.cpp',
   'test/src/RTC/TestSubchannelsCodec.cpp',

package/worker/mocks/include/MockShared.hpp

@@ -60,6 +60,13 @@ namespace mocks
 			return static_cast<int64_t>(GetTimeUs());
 		}
 
+		// NOTE: The NTP epoch is made to be the very clock given by argument, so that tests
+		// can reason about a single set of values.
+		uint64_t GetNtpOffsetMs() override
+		{
+			return 0;
+		}
+
 		// Methods for testing.
 	public:
 		MockBackoffTimerHandle* GetBackoffTimer(const std::string_view label) const

package/worker/src/DepLibUV.cpp

@@ -3,10 +3,12 @@
 
 #include "DepLibUV.hpp"
 #include "Logger.hpp"
+#include "Utils.hpp"
 
 /* Class variables. */
 
 thread_local uv_loop_t* DepLibUV::loop{ nullptr };
+thread_local uint64_t DepLibUV::ntpOffsetMs{ 0 };
 
 /* Static methods for UV callbacks. */
 
@@ -48,6 +50,21 @@ void DepLibUV::ClassInit()
 	{
 		MS_ABORT("libuv loop initialization failed");
 	}
+
+	// Take the distance from our own monotonic clock to the NTP epoch just once, so that
+	// the NTP timestamps we generate never step when the system clock is adjusted.
+	uv_timeval64_t timeval{}; // NOLINT(cppcoreguidelines-pro-type-member-init)
+
+	if (uv_gettimeofday(std::addressof(timeval)) != 0)
+	{
+		MS_ABORT("uv_gettimeofday() failed");
+	}
+
+	const auto unixSec = static_cast<uint64_t>(timeval.tv_sec);
+	const auto unixMs  = (unixSec * 1000) + (static_cast<uint64_t>(timeval.tv_usec) / 1000);
+	const auto ntpMs   = unixMs + (static_cast<uint64_t>(Utils::Time::UnixNtpOffsetSec) * 1000);
+
+	DepLibUV::ntpOffsetMs = ntpMs - DepLibUV::GetTimeMs();
 }
 
 void DepLibUV::ClassDestroy()

package/worker/src/RTC/Consumer.cpp

@@ -1414,22 +1414,25 @@ namespace RTC
 		return true;
 	}
 
-	void Consumer::NeedWorstRemoteFractionLost(uint32_t /*mappedSsrc*/, uint8_t& worstRemoteFractionLost)
+	uint8_t Consumer::GetWorstRemoteFractionLost(uint32_t /*mappedSsrc*/) const
 	{
 		MS_TRACE();
 
 		if (!IsActive())
 		{
-			return;
+			return 0;
 		}
 
-		for (auto* rtpStream : this->rtpStreams)
+		uint8_t worstRemoteFractionLost{ 0 };
+
+		for (const auto* rtpStream : this->rtpStreams)
 		{
-			auto fractionLost = rtpStream->GetFractionLost();
+			const auto fractionLost = rtpStream->GetFractionLost();
 
-			// If our fraction lost is worse than the given one, update it.
 			worstRemoteFractionLost = std::max(fractionLost, worstRemoteFractionLost);
 		}
+
+		return worstRemoteFractionLost;
 	}
 
 	void Consumer::ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket)

package/worker/src/RTC/Producer.cpp

@@ -580,6 +580,9 @@ namespace RTC
 
 		ReceiveRtpPacketResult result;
 		bool isRtx{ false };
+		// Highest RTP timestamp of the stream before this packet is processed, so that it
+		// can be told afterwards whether this packet made it change.
+		const auto previousMaxPacketTs = rtpStream->GetMaxPacketTs();
 
 		// Media packet.
 		if (packet->GetSsrc() == rtpStream->GetSsrc())
@@ -675,7 +678,10 @@ namespace RTC
 		}
 
 		// Post-process the packet.
-		PostProcessRtpPacket(packet);
+		PostProcessRtpPacket(
+		  packet,
+		  rtpStream,
+		  /*maxPacketTsChanged*/ rtpStream->GetMaxPacketTs() != previousMaxPacketTs);
 
 		this->listener->OnProducerRtpPacketReceived(this, packet);
 
@@ -691,7 +697,7 @@ namespace RTC
 		if (it != this->mapSsrcRtpStream.end())
 		{
 			auto* rtpStream  = it->second;
-			const bool first = rtpStream->GetSenderReportNtpMs() == 0;
+			const bool first = !rtpStream->GetSenderReportMapping().has_value();
 
 			rtpStream->ReceiveRtcpSenderReport(report);
 
@@ -765,7 +771,8 @@ namespace RTC
 		// Add a receiver reference time report if no present in the packet.
 		if (!packet->HasReceiverReferenceTime())
 		{
-			auto ntp                    = Utils::Time::TimeMs2Ntp(nowMs);
+			auto ntp = Utils::Time::TimeMs2Ntp(nowMs + this->shared->GetNtpOffsetMs());
+
 			receiverReferenceTimeReport = new RTC::RTCP::ReceiverReferenceTime();
 
 			receiverReferenceTimeReport->SetNtpSec(ntp.seconds);
@@ -1223,19 +1230,64 @@ namespace RTC
 			}
 
 			// Proxy http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time.
+			//
+			// The capture timestamp is left untouched since it belongs to the clock of the
+			// capture system, but the capture clock offset has to be rewritten. It tells the
+			// offset between the clock of the capture system and the clock of whoever sends
+			// the stream, and by forwarding the stream we become that sender:
+			//
+			//   capture NTP clock = sender NTP clock + capture clock offset
+			//
+			// @see https://datatracker.ietf.org/doc/html/draft-ietf-avtcore-abs-capture-time-00
 			extenValue = packet->GetExtensionValue(this->rtpHeaderExtensionIds.absCaptureTime, extenLen);
 
-			if (extenValue)
+			// NOTE: The extension value is 8 or 16 bytes long depending on whether it holds
+			// the capture clock offset or not.
+			if (extenValue && (extenLen == 8u || extenLen == 16u))
 			{
-				std::memcpy(bufferPtr, extenValue, extenLen);
+				// Offset between our own clock and the clock of the sender of this stream, in
+				// the format the extension uses.
+				//
+				// NOTE: While it cannot be told yet, and it never can if the sender sends no
+				// RTCP at all, both clocks are taken to be the same. Our own clock is real NTP
+				// and so is the one of any sender that fills this extension, so no offset is
+				// the most likely case rather than a wild guess.
+				const auto remoteClockOffsetMs = this->listener->OnProducerNeedRemoteClockOffsetMs(this);
+				const auto remoteClockOffsetQ32x32 =
+				  Utils::Time::TimeMs2Q32x32(remoteClockOffsetMs.value_or(0));
+
+				// NOTE: An offset that does not fit in the extension means a sender whose clock
+				// is decades away from ours, and there is no value to write that would not be a
+				// lie, so the extension is not forwarded.
+				if (remoteClockOffsetQ32x32.has_value())
+				{
+					const auto absCaptureTimestamp = Utils::Byte::Get8Bytes(extenValue, 0);
+					// Offset between the clock of the capture system and the clock of the sender
+					// of this stream, as told by that sender. There is none when the extension
+					// comes in its shortened form, which is what a sender that captures its own
+					// media sends.
+					const int64_t incomingCaptureClockOffset =
+					  extenLen == 16u ? static_cast<int64_t>(Utils::Byte::Get8Bytes(extenValue, 8)) : 0;
+					// NOTE: The estimator gives our own clock minus the one of the sender, so
+					// subtracting it adds the offset of the sender against ours.
+					const int64_t captureClockOffset =
+					  incomingCaptureClockOffset - remoteClockOffsetQ32x32.value();
+
+					// The extension is always forwarded in its extended form, since the capture
+					// clock offset is always written.
+					extenLen = 16u;
+
+					Utils::Byte::Set8Bytes(bufferPtr, 0, absCaptureTimestamp);
+					Utils::Byte::Set8Bytes(bufferPtr, 8, static_cast<uint64_t>(captureClockOffset));
 
-				extensions.emplace_back(
-				  /*type*/ RTC::RtpHeaderExtensionUri::Type::ABS_CAPTURE_TIME,
-				  /*id*/ static_cast<uint8_t>(RTC::RtpHeaderExtensionUri::Type::ABS_CAPTURE_TIME),
-				  /*len*/ extenLen,
-				  /*value*/ bufferPtr);
+					extensions.emplace_back(
+					  /*type*/ RTC::RtpHeaderExtensionUri::Type::ABS_CAPTURE_TIME,
+					  /*id*/ static_cast<uint8_t>(RTC::RtpHeaderExtensionUri::Type::ABS_CAPTURE_TIME),
+					  /*len*/ extenLen,
+					  /*value*/ bufferPtr);
 
-				bufferPtr += extenLen;
+					bufferPtr += extenLen;
+				}
 			}
 
 			// Proxy http://www.webrtc.org/experiments/rtp-hdrext/playout-delay
@@ -1409,10 +1461,28 @@ namespace RTC
 		return true;
 	}
 
-	inline void Producer::PostProcessRtpPacket(RTC::RTP::Packet* packet)
+	inline void Producer::PostProcessRtpPacket(
+	  RTC::RTP::Packet* packet, RTC::RTP::RtpStreamRecv* rtpStream, bool maxPacketTsChanged)
 	{
 		MS_TRACE();
 
+		// Store the capture instant of the packet holding the highest RTP timestamp of its
+		// stream, which is the one the Sender Reports we send are built upon.
+		if (maxPacketTsChanged)
+		{
+			const auto captureMs =
+			  this->listener->OnProducerNeedLocalCaptureMs(this, rtpStream, packet->GetTimestamp());
+
+			if (captureMs.has_value())
+			{
+				packet->SetCaptureMs(captureMs.value());
+
+				// Also let the stream remember it, since this is the only place where the
+				// capture instant of a received packet can be told.
+				rtpStream->SetCaptureMapping(captureMs.value(), packet->GetTimestamp());
+			}
+		}
+
 		if (this->kind == RTC::Media::Kind::VIDEO)
 		{
 			bool camera{ false };
@@ -1697,15 +1767,14 @@ namespace RTC
 		this->listener->OnProducerSendRtcpPacket(this, packet);
 	}
 
-	inline void Producer::OnRtpStreamNeedWorstRemoteFractionLost(
-	  RTC::RTP::RtpStreamRecv* rtpStream, uint8_t& worstRemoteFractionLost)
+	inline uint8_t Producer::OnRtpStreamNeedWorstRemoteFractionLost(RTC::RTP::RtpStreamRecv* rtpStream)
 	{
 		MS_TRACE();
 
-		auto mappedSsrc = this->mapRtpStreamMappedSsrc.at(rtpStream);
+		const auto mappedSsrc = this->mapRtpStreamMappedSsrc.at(rtpStream);
 
 		// Notify the listener.
-		this->listener->OnProducerNeedWorstRemoteFractionLost(this, mappedSsrc, worstRemoteFractionLost);
+		return this->listener->OnProducerNeedWorstRemoteFractionLost(this, mappedSsrc);
 	}
 
 	inline void Producer::OnKeyFrameNeeded(

package/worker/src/RTC/RTP/Packet.cpp

@@ -391,6 +391,11 @@ namespace RTC
 			MS_DUMP_CLEAN(indentation, "  padding length: %" PRIu8, GetPaddingLength());
 			MS_DUMP_CLEAN(indentation, "  padded to 4 bytes: %s", IsPaddedTo4Bytes() ? "yes" : "no");
 
+			if (GetCaptureMs())
+			{
+				MS_DUMP_CLEAN(indentation, "  capture time (ms):%" PRIu64, GetCaptureMs().value());
+			}
+
 			if (this->payloadDescriptorHandler)
 			{
 				MS_DUMP_CLEAN(indentation + 1, "<PayloadDescriptorHandler>");
@@ -422,6 +427,9 @@ namespace RTC
 			// Clone extension ids.
 			clonedPacket->headerExtensionIds = this->headerExtensionIds;
 
+			// Clone capture time.
+			clonedPacket->captureMs = this->captureMs;
+
 			// Assign the payload descriptor handler.
 			clonedPacket->payloadDescriptorHandler = this->payloadDescriptorHandler;
 
@@ -1174,7 +1182,7 @@ namespace RTC
 			// Extension value can be 8 or 16 bytes depending on whether it contains
 			// estimated capture clock offset or not.
 			//
-			// https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/abs-capture-time
+			// @see https://datatracker.ietf.org/doc/html/draft-ietf-avtcore-abs-capture-time-00
 			if (!extenValue || (extenLen != 8u && extenLen != 16u))
 			{
 				return false;

package/worker/src/RTC/RTP/RtpStream.cpp

@@ -137,10 +137,11 @@ namespace RTC
 			{
 				InitSeq(seq);
 
-				this->started     = true;
-				this->maxSeq      = seq - 1;
-				this->maxPacketTs = packet->GetTimestamp();
-				this->maxPacketMs = this->shared->GetTimeMs();
+				this->started            = true;
+				this->maxSeq             = seq - 1;
+				this->maxPacketTs        = packet->GetTimestamp();
+				this->maxPacketMs        = this->shared->GetTimeMs();
+				this->maxPacketCaptureMs = packet->GetCaptureMs();
 			}
 
 			// If not a valid packet ignore it.
@@ -158,8 +159,9 @@ namespace RTC
 			// Update highest seen RTP timestamp.
 			if (Utils::Number::IsHigherThan<uint32_t>(packet->GetTimestamp(), this->maxPacketTs))
 			{
-				this->maxPacketTs = packet->GetTimestamp();
-				this->maxPacketMs = this->shared->GetTimeMs();
+				this->maxPacketTs        = packet->GetTimestamp();
+				this->maxPacketMs        = this->shared->GetTimeMs();
+				this->maxPacketCaptureMs = packet->GetCaptureMs();
 			}
 
 			return true;
@@ -226,8 +228,9 @@ namespace RTC
 					  this->maxPacketTs,
 					  packet->GetTimestamp());
 
-					this->maxPacketTs = packet->GetTimestamp();
-					this->maxPacketMs = this->shared->GetTimeMs();
+					this->maxPacketTs        = packet->GetTimestamp();
+					this->maxPacketMs        = this->shared->GetTimeMs();
+					this->maxPacketCaptureMs = packet->GetCaptureMs();
 				}
 			}
 			// Too old packet received (older than the allowed misorder).
@@ -248,8 +251,9 @@ namespace RTC
 
 					InitSeq(seq);
 
-					this->maxPacketTs = packet->GetTimestamp();
-					this->maxPacketMs = this->shared->GetTimeMs();
+					this->maxPacketTs        = packet->GetTimestamp();
+					this->maxPacketMs        = this->shared->GetTimeMs();
+					this->maxPacketCaptureMs = packet->GetCaptureMs();
 
 					// Notify the subclass about it.
 					UserOnSequenceNumberReset();