mediasoup 3.26.0 → 3.27.1

package/worker/src/RTC/SimulcastProducerStreamManager.cpp

@@ -8,10 +8,10 @@ namespace RTC
 {
 	/* Static. */
 
-	static constexpr uint64_t StreamMinActiveMs{ 2000u };
-	static constexpr uint64_t BweDowngradeConservativeMs{ 10000u };
-	static constexpr uint64_t BweDowngradeMinActiveMs{ 8000u };
-	static constexpr uint16_t MaxSequenceNumberGap{ 100u };
+	static constexpr int64_t StreamMinActiveMs{ 2000 };
+	static constexpr int64_t BweDowngradeConservativeMs{ 10000 };
+	static constexpr int64_t BweDowngradeMinActiveMs{ 8000 };
+	static constexpr uint16_t MaxSequenceNumberGap{ 100 };
 
 	/* Instance methods. */
 
@@ -200,18 +200,18 @@ namespace RTC
 		}
 	}
 
-	uint32_t SimulcastProducerStreamManager::IncreaseLayer(
-	  uint32_t bitrate, bool considerLoss, float lossPercentage, uint64_t nowMs)
+	int64_t SimulcastProducerStreamManager::IncreaseLayer(
+	  int64_t bitrate, bool considerLoss, float lossPercentage, int64_t nowMs)
 	{
 		MS_TRACE();
 
 		// If already in the preferred layers, do nothing.
 		if (this->provisionalTargetLayers == this->preferredLayers)
 		{
-			return 0u;
+			return 0;
 		}
 
-		uint32_t virtualBitrate;
+		int64_t virtualBitrate;
 
 		if (considerLoss)
 		{
@@ -235,7 +235,7 @@ namespace RTC
 			virtualBitrate = bitrate;
 		}
 
-		uint32_t requiredBitrate{ 0u };
+		int64_t requiredBitrate{ 0 };
 		int16_t spatialLayer{ 0 };
 		int16_t temporalLayer{ 0 };
 		// Whether a usable spatial layer has been found, in which case we must not
@@ -351,7 +351,7 @@ namespace RTC
 				// temporal spatial layer if this is the temporal layer 0 of a higher
 				// spatial layer.
 				if (
-				  requiredBitrate && temporalLayer == 0 && this->provisionalTargetLayers.spatial > -1 &&
+				  requiredBitrate > 0 && temporalLayer == 0 && this->provisionalTargetLayers.spatial > -1 &&
 				  spatialLayer > this->provisionalTargetLayers.spatial)
 				{
 					auto* provisionalProducerRtpStream =
@@ -365,13 +365,13 @@ namespace RTC
 					}
 					else
 					{
-						requiredBitrate = 1u; // Don't set 0 since it would be ignored.
+						requiredBitrate = 1; // Don't set 0 since it would be ignored.
 					}
 				}
 
 				MS_DEBUG_DEV(
-				  "testing layers %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32
-				  ", required bitrate:%" PRIu32 "]",
+				  "testing layers %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIi64
+				  ", required bitrate:%" PRIi64 "]",
 				  spatialLayer,
 				  temporalLayer,
 				  virtualBitrate,
@@ -379,7 +379,7 @@ namespace RTC
 
 				// If active layer, end iterations here. Otherwise move to next spatial
 				// layer.
-				if (requiredBitrate)
+				if (requiredBitrate > 0)
 				{
 					goto done;
 				}
@@ -399,15 +399,15 @@ namespace RTC
 	done:
 
 		// No higher active layers found.
-		if (!requiredBitrate)
+		if (requiredBitrate <= 0)
 		{
-			return 0u;
+			return 0;
 		}
 
 		// No luck.
 		if (requiredBitrate > virtualBitrate)
 		{
-			return 0u;
+			return 0;
 		}
 
 		// Set provisional layers.
@@ -415,8 +415,8 @@ namespace RTC
 		this->provisionalTargetLayers.temporal = temporalLayer;
 
 		MS_DEBUG_DEV(
-		  "setting provisional layers to %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32
-		  ", required bitrate:%" PRIu32 "]",
+		  "setting provisional layers to %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIi64
+		  ", required bitrate:%" PRIi64 "]",
 		  this->provisionalTargetLayers.spatial,
 		  this->provisionalTargetLayers.temporal,
 		  virtualBitrate,
@@ -436,7 +436,7 @@ namespace RTC
 		}
 	}
 
-	void SimulcastProducerStreamManager::ApplyLayers(uint64_t rtpStreamActiveMs)
+	void SimulcastProducerStreamManager::ApplyLayers(int64_t rtpStreamActiveMs)
 	{
 		MS_TRACE();
 
@@ -467,11 +467,11 @@ namespace RTC
 		}
 	}
 
-	uint32_t SimulcastProducerStreamManager::GetDesiredBitrate(uint64_t nowMs) const
+	int64_t SimulcastProducerStreamManager::GetDesiredBitrate(int64_t nowMs) const
 	{
 		MS_TRACE();
 
-		uint32_t desiredBitrate{ 0u };
+		int64_t desiredBitrate{ 0 };
 
 		// Let's iterate all streams of the Producer (from highest to lowest) and
 		// obtain their bitrate. Choose the highest one.
@@ -673,13 +673,13 @@ namespace RTC
 				else
 				{
 					// Calculate capture instant and TS stuff.
-					const auto captureMs1 = tsReferenceCaptureMapping.value().captureMs;
-					const auto ts1        = tsReferenceCaptureMapping.value().ts;
-					const auto captureMs2 = captureMapping.value().captureMs;
-					const auto ts2        = captureMapping.value().ts;
-					const int64_t diffMs = static_cast<int64_t>(captureMs2) - static_cast<int64_t>(captureMs1);
-					const int64_t diffTs  = diffMs * clockRate / 1000;
-					const uint32_t newTs2 = ts2 - diffTs;
+					const auto captureAtUs1 = tsReferenceCaptureMapping.value().captureAtUs;
+					const auto ts1          = tsReferenceCaptureMapping.value().ts;
+					const auto captureAtUs2 = captureMapping.value().captureAtUs;
+					const auto ts2          = captureMapping.value().ts;
+					const int64_t diffUs    = captureAtUs2 - captureAtUs1;
+					const int64_t diffTs    = (diffUs * clockRate) / 1000000;
+					const uint32_t newTs2   = ts2 - diffTs;
 
 					// Apply offset. This is the difference that later must be removed from
 					// the sending RTP packet.
@@ -695,15 +695,17 @@ namespace RTC
 			{
 				// Max delay in ms we allow for the stream when switching.
 				// https://en.wikipedia.org/wiki/Audio-to-video_synchronization#Recommendations
-				static constexpr uint32_t MaxExtraOffsetMs{ 75 };
+				static constexpr int64_t MaxExtraOffsetMs{ 75 };
 
 				// Outgoing packet matches the highest timestamp seen in the previous
 				// stream. Apply an expected offset for a new frame in a 30fps stream.
-				static constexpr uint8_t MsOffset{ 33 }; // (1 / 30 * 1000).
+				static constexpr int64_t OffsetMs{ 33 }; // (1 / 30 * 1000).
 
 				const int64_t maxTsExtraOffset = MaxExtraOffsetMs * clockRate / 1000;
-				uint32_t tsExtraOffset =
-				  maxPacketTs - packet->GetTimestamp() + tsOffset + (MsOffset * clockRate / 1000);
+
+				// NOTE: RTP timestamps wrap around, so the sum is truncated on purpose.
+				auto tsExtraOffset = static_cast<uint32_t>(
+				  maxPacketTs - packet->GetTimestamp() + tsOffset + (OffsetMs * clockRate / 1000));
 
 				// NOTE: Don't ask for a key frame if already done.
 				if (this->keyFrameForTsOffsetRequested)
@@ -717,7 +719,7 @@ namespace RTC
 						  "which still too high RTP timestamp extra offset is needed (%" PRIu32 ")",
 						  tsExtraOffset);
 
-						tsExtraOffset = 1u;
+						tsExtraOffset = 1;
 					}
 				}
 				else if (std::cmp_greater(tsExtraOffset, maxTsExtraOffset))
@@ -746,7 +748,7 @@ namespace RTC
 					return result;
 				}
 
-				if (tsExtraOffset > 0u)
+				if (tsExtraOffset > 0)
 				{
 					MS_DEBUG_TAG(
 					  simulcast,
@@ -918,7 +920,7 @@ namespace RTC
 		if (
 		  newTargetSpatialLayer != -1 &&
 		  (this->tsReferenceSpatialLayer == -1 ||
-		   !GetProducerTsReferenceRtpStream()->GetCaptureMapping().has_value()))
+			 !GetProducerTsReferenceRtpStream()->GetCaptureMapping().has_value()))
 		{
 			MS_DEBUG_TAG(
 			  simulcast, "using spatial layer %" PRIi16 " as RTP timestamp reference", newTargetSpatialLayer);
@@ -982,7 +984,7 @@ namespace RTC
 		// Start with no layers.
 		newTargetLayers.Reset();
 
-		auto nowMs = this->shared->GetTimeMs();
+		const int64_t nowMs = this->shared->GetTimeMs();
 
 		for (size_t sIdx{ 0u }; sIdx < this->producerRtpStreams.size(); ++sIdx)
 		{

package/worker/src/RTC/SvcProducerStreamManager.cpp

@@ -8,8 +8,8 @@ namespace RTC
 {
 	/* Static. */
 
-	static constexpr uint64_t BweDowngradeConservativeMs{ 10000u };
-	static constexpr uint64_t BweDowngradeMinActiveMs{ 8000u };
+	static constexpr int64_t BweDowngradeConservativeMs{ 10000 };
+	static constexpr int64_t BweDowngradeMinActiveMs{ 8000 };
 
 	/* Instance methods. */
 
@@ -122,23 +122,37 @@ namespace RTC
 		// Do nothing.
 	}
 
-	uint32_t SvcProducerStreamManager::IncreaseLayer(
-	  uint32_t bitrate, bool considerLoss, float lossPercentage, uint64_t nowMs)
+	void SvcProducerStreamManager::ProducerSpatialLayerActivityChanged(
+	  RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint8_t /*spatialLayer*/, bool /*isActive*/)
 	{
 		MS_TRACE();
 
-		if (!this->producerRtpStream || this->producerRtpStream->GetScore() == 0u)
+		// All spatial layers share a single RTP stream, so neither its score nor its
+		// inactivity check can tell that just one of them started or stopped. This is
+		// the only chance to reconsider the target layers.
+		if (IsActive())
+		{
+			MayChangeLayers(/*force*/ false);
+		}
+	}
+
+	int64_t SvcProducerStreamManager::IncreaseLayer(
+	  int64_t bitrate, bool considerLoss, float lossPercentage, int64_t nowMs)
+	{
+		MS_TRACE();
+
+		if (!this->producerRtpStream || this->producerRtpStream->GetScore() == 0)
 		{
-			return 0u;
+			return 0;
 		}
 
 		// If already in the preferred layers, do nothing.
 		if (this->provisionalTargetLayers == this->preferredLayers)
 		{
-			return 0u;
+			return 0;
 		}
 
-		uint32_t virtualBitrate;
+		int64_t virtualBitrate;
 
 		if (considerLoss)
 		{
@@ -162,7 +176,7 @@ namespace RTC
 			virtualBitrate = bitrate;
 		}
 
-		uint32_t requiredBitrate{ 0u };
+		int64_t requiredBitrate{ 0 };
 		int16_t spatialLayer{ 0 };
 		int16_t temporalLayer{ 0 };
 
@@ -215,7 +229,7 @@ namespace RTC
 				// layer if the new layer is the temporal layer 0 of a higher spatial
 				// layer.
 				if (
-				  this->encodingContext->IsKSvc() && requiredBitrate && temporalLayer == 0 &&
+				  this->encodingContext->IsKSvc() && requiredBitrate > 0 && temporalLayer == 0 &&
 				  this->provisionalTargetLayers.spatial > -1 &&
 				  spatialLayer > this->provisionalTargetLayers.spatial)
 				{
@@ -228,13 +242,13 @@ namespace RTC
 					}
 					else
 					{
-						requiredBitrate = 1u; // Don't set 0 since it would be ignored.
+						requiredBitrate = 1; // Don't set 0 since it would be ignored.
 					}
 				}
 
 				MS_DEBUG_DEV(
-				  "testing layers %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32
-				  ", required bitrate:%" PRIu32 "]",
+				  "testing layers %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIi64
+				  ", required bitrate:%" PRIi64 "]",
 				  spatialLayer,
 				  temporalLayer,
 				  virtualBitrate,
@@ -242,7 +256,7 @@ namespace RTC
 
 				// If active layer, end iterations here. Otherwise move to next spatial
 				// layer.
-				if (requiredBitrate)
+				if (requiredBitrate > 0)
 				{
 					goto done;
 				}
@@ -263,15 +277,15 @@ namespace RTC
 	done:
 
 		// No higher active layers found.
-		if (!requiredBitrate)
+		if (requiredBitrate <= 0)
 		{
-			return 0u;
+			return 0;
 		}
 
 		// No luck.
 		if (requiredBitrate > virtualBitrate)
 		{
-			return 0u;
+			return 0;
 		}
 
 		// Set provisional layers.
@@ -279,8 +293,8 @@ namespace RTC
 		this->provisionalTargetLayers.temporal = temporalLayer;
 
 		MS_DEBUG_DEV(
-		  "upgrading to layers %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32
-		  ", required bitrate:%" PRIu32 "]",
+		  "upgrading to layers %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIi64
+		  ", required bitrate:%" PRIi64 "]",
 		  this->provisionalTargetLayers.spatial,
 		  this->provisionalTargetLayers.temporal,
 		  virtualBitrate,
@@ -300,7 +314,7 @@ namespace RTC
 		}
 	}
 
-	void SvcProducerStreamManager::ApplyLayers(uint64_t rtpStreamActiveMs)
+	void SvcProducerStreamManager::ApplyLayers(int64_t rtpStreamActiveMs)
 	{
 		MS_TRACE();
 
@@ -332,16 +346,16 @@ namespace RTC
 		}
 	}
 
-	uint32_t SvcProducerStreamManager::GetDesiredBitrate(uint64_t nowMs) const
+	int64_t SvcProducerStreamManager::GetDesiredBitrate(int64_t nowMs) const
 	{
 		MS_TRACE();
 
-		if (!this->producerRtpStream || this->producerRtpStream->GetScore() == 0u)
+		if (!this->producerRtpStream || this->producerRtpStream->GetScore() == 0)
 		{
-			return 0u;
+			return 0;
 		}
 
-		uint32_t desiredBitrate{ 0u };
+		int64_t desiredBitrate{ 0 };
 
 		// When using K-SVC each spatial layer is independent of the others.
 		if (this->encodingContext->IsKSvc())
@@ -564,7 +578,7 @@ namespace RTC
 		// Start with no layers.
 		newTargetLayers.Reset();
 
-		auto nowMs = this->shared->GetTimeMs();
+		const int64_t nowMs = this->shared->GetTimeMs();
 		int16_t spatialLayer{ 0 };
 
 		if (!this->producerRtpStream)

package/worker/src/RTC/TcpConnection.cpp

@@ -19,8 +19,8 @@ namespace RTC
 
 	/* Instance methods. */
 
-	TcpConnection::TcpConnection(Listener* listener, size_t bufferSize)
-	  : ::TcpConnectionHandle::TcpConnectionHandle(bufferSize), listener(listener)
+	TcpConnection::TcpConnection(Listener* listener, SharedInterface* shared, size_t bufferSize)
+	  : ::TcpConnectionHandle::TcpConnectionHandle(bufferSize), listener(listener), shared(shared)
 	{
 		MS_TRACE();
 	}
@@ -34,6 +34,11 @@ namespace RTC
 	{
 		MS_TRACE();
 
+		// NOTE: Take the arrival time before anything else is done with the read
+		// data, so that it doesn't include the cost of processing it. All the frames
+		// found within this read share it, since they all arrived in it.
+		const int64_t receivedAtUs = this->shared->GetTimeUs();
+
 		MS_DEBUG_DEV(
 		  "data received [local:%s :%" PRIu16 ", remote:%s :%" PRIu16 "]",
 		  GetLocalIp().c_str(),
@@ -88,7 +93,8 @@ namespace RTC
 					// later.
 					std::memcpy(ReadBuffer, packet, packetLen);
 
-					this->listener->OnTcpConnectionPacketReceived(this, ReadBuffer, packetLen, ReadBufferSize);
+					this->listener->OnTcpConnectionPacketReceived(
+					  this, ReadBuffer, packetLen, ReadBufferSize, receivedAtUs);
 				}
 
 				// If there is no more space available in the buffer and that is because

package/worker/src/RTC/TcpServer.cpp

@@ -17,6 +17,7 @@ namespace RTC
 	TcpServer::TcpServer(
 	  Listener* listener,
 	  RTC::TcpConnection::Listener* connListener,
+	  SharedInterface* shared,
 	  std::string& ip,
 	  uint16_t port,
 	  RTC::Transport::SocketFlags& flags)
@@ -24,6 +25,7 @@ namespace RTC
 	    ::TcpServerHandle::TcpServerHandle(RTC::PortManager::BindTcp(ip, port, flags)),
 	    listener(listener),
 	    connListener(connListener),
+	    shared(shared),
 	    fixedPort(true)
 	{
 		MS_TRACE();
@@ -32,6 +34,7 @@ namespace RTC
 	TcpServer::TcpServer(
 	  Listener* listener,
 	  RTC::TcpConnection::Listener* connListener,
+	  SharedInterface* shared,
 	  std::string& ip,
 	  uint16_t minPort,
 	  uint16_t maxPort,
@@ -41,7 +44,8 @@ namespace RTC
 	    ::TcpServerHandle::TcpServerHandle(
 	      RTC::PortManager::BindTcp(ip, minPort, maxPort, flags, portRangeKey)),
 	    listener(listener),
-	    connListener(connListener)
+	    connListener(connListener),
+	    shared(shared)
 	{
 		MS_TRACE();
 
@@ -63,7 +67,8 @@ namespace RTC
 		MS_TRACE();
 
 		// Allocate a new RTC::TcpConnection for the TcpServer to handle it.
-		auto* connection = new RTC::TcpConnection(this->connListener, TcpConnectionBufferSize);
+		auto* connection =
+		  new RTC::TcpConnection(this->connListener, this->shared, TcpConnectionBufferSize);
 
 		// Accept it.
 		AcceptTcpConnection(connection);