mediasoup 3.28.0 → 3.29.0

package/worker/include/RTC/BWE/SenderTransportCongestionController.hpp

@@ -0,0 +1,343 @@
+#ifndef MS_RTC_BWE_SENDER_TRANSPORT_CONGESTION_CONTROLLER_HPP
+#define MS_RTC_BWE_SENDER_TRANSPORT_CONGESTION_CONTROLLER_HPP
+
+#include "common.hpp"
+#include "handles/TimerHandle.hpp"
+#include "RTC/BWE/AlrDetector.hpp"
+#include "RTC/BWE/BweTypes.hpp"
+#include "RTC/BWE/DelayBasedBwe.hpp"
+#include "RTC/BWE/FeedbackAdapter.hpp"
+#include "RTC/BWE/ProbeBitrateEstimator.hpp"
+#include "RTC/BWE/ProbeController.hpp"
+#include "RTC/BWE/ProbingScheduler.hpp"
+#include "RTC/BWE/RobustThroughputEstimator.hpp"
+#include "RTC/BWE/SendPacketHistory.hpp"
+#include "RTC/BWE/TargetRateController.hpp"
+#include "RTC/Consts.hpp"
+#include "RTC/RTCP/FeedbackRtpTransport.hpp"
+#include "RTC/RTP/Packet.hpp"
+#include "SharedInterface.hpp"
+#include <vector>
+
+namespace RTC
+{
+	namespace BWE
+	{
+		/**
+		 * Everything the sending side of a transport does about congestion, which
+		 * amounts to deciding how much may be sent towards the remote endpoint.
+		 *
+		 * It owns the whole control loop and is, along with its receiving
+		 * counterpart, the only part of the module that the transport deals with.
+		 * What comes in is what leaves this endpoint and what the remote one reports
+		 * back about it; what comes out is a target bitrate, for whoever shares it
+		 * among the streams being sent, and the packets of a probe when the link has
+		 * to be measured beyond what the media alone can tell.
+		 */
+		class SenderTransportCongestionController : public ProbingScheduler::Listener,
+		                                            public TimerHandleInterface::Listener
+		{
+		public:
+			class Listener
+			{
+			public:
+				virtual ~Listener() = default;
+
+			public:
+				/**
+				 * The bitrate that may be sent towards the remote endpoint changed
+				 * (bps).
+				 */
+				virtual void OnSenderTransportCongestionControllerTargetBitrate(
+				  SenderTransportCongestionController* senderTransportCongestionController,
+				  int64_t targetBitrate) = 0;
+
+				/**
+				 * A packet of a probe is ready to go out.
+				 *
+				 * @param sequenceNumber - The one given to the packet, which has to be
+				 *   given back through `OnRtpPacketSent()` once the packet has actually
+				 *   left, just like for a packet of media.
+				 *
+				 * @remarks
+				 * - The packet must be sent before returning, since the next one is
+				 *   built over the very same instance.
+				 */
+				virtual void OnSenderTransportCongestionControllerSendRtpProbePacket(
+				  SenderTransportCongestionController* senderTransportCongestionController,
+				  RTC::RTP::Packet* packet,
+				  int64_t sequenceNumber) = 0;
+			};
+
+			/**
+			 * What a packet about to be sent cannot tell about itself.
+			 */
+			struct RtpPacketToBeSentOptions
+			{
+				/**
+				 * Whether it belongs to an audio stream.
+				 */
+				bool isAudio{ false };
+				/**
+				 * Whether it is being sent again after having been reported lost.
+				 */
+				bool isRetransmission{ false };
+				/**
+				 * SSRC of the stream it retransmits, when it goes out over RTX.
+				 */
+				std::optional<uint32_t> originalSsrc;
+			};
+
+			struct SenderTransportCongestionControllerOptions
+			{
+				/**
+				 * Bitrate the control loop starts from, before anything has been
+				 * measured (bps). It only means something until the first feedback
+				 * arrives, which is why it is given once and never changed: starting
+				 * over would mean throwing away what has been learnt since.
+				 */
+				int64_t startBitrate{ 300000 };
+				/**
+				 * Least that may ever be estimated (bps). It exists so that the loop
+				 * cannot drive its own estimate to nothing, which would be
+				 * unrecoverable: with nothing to hand out, nothing is sent, so there is
+				 * nothing to measure and no way back up.
+				 */
+				int64_t minBitrate{ RTC::Consts::BweMinBitrate };
+				/**
+				 * Most that may ever be estimated (bps), or `Types::BitrateInfinite` to
+				 * let the link decide.
+				 */
+				int64_t maxBitrate{ Types::BitrateInfinite };
+			};
+
+		private:
+			/**
+			 * What a call leaves to be done once it is over, so that nothing of this
+			 * class is half way through anything by the time the listener is told and
+			 * calls back in.
+			 */
+			struct PendingUpdate
+			{
+				/**
+				 * Bursts that were asked for, which are handed to the scheduler all at
+				 * once.
+				 */
+				std::vector<Types::ProbeClusterConfig> probeClusterConfigs;
+				/**
+				 * Target the listener is to be told about, when something it reasons
+				 * about turned out to have changed.
+				 *
+				 * @remarks
+				 * - It is worked out where the call would have announced it, and only
+				 *   handed over at the end, since what is reconsidered along the way
+				 *   depends on the order the original does things in.
+				 */
+				std::optional<int64_t> targetBitrateToNotify;
+			};
+
+		public:
+			SenderTransportCongestionController(
+			  Listener* listener,
+			  SharedInterface* shared,
+			  SenderTransportCongestionControllerOptions options);
+
+			~SenderTransportCongestionController() override = default;
+
+			SenderTransportCongestionController(const SenderTransportCongestionController&) = delete;
+			SenderTransportCongestionController& operator=(const SenderTransportCongestionController&) = delete;
+
+			/**
+			 * Tell whether the transport can carry anything at all. Nothing is probed
+			 * until it can, so forgetting this leaves the initial probing waiting
+			 * forever.
+			 */
+			void SetNetworkAvailable(bool networkAvailable);
+
+			/**
+			 * Take note of a packet that is about to be sent, and write into it the
+			 * transport wide sequence number and the abs-send-time that go on the
+			 * wire with it.
+			 *
+			 * @returns The sequence number given to the packet, which has to be given
+			 *   back once the packet has actually left, or no value when the packet
+			 *   carries no room for it. A packet without that extension cannot be
+			 *   reported on, so nothing is taken note of and only its bytes count.
+			 */
+			std::optional<int64_t> OnRtpPacketToBeSent(
+			  RTC::RTP::Packet* packet, const RtpPacketToBeSentOptions& options);
+
+			/**
+			 * Feed the confirmation that a packet left through the socket.
+			 *
+			 * @param sequenceNumber - What `OnRtpPacketToBeSent()` returned for it.
+			 * @param size - Length of the packet (bytes), which is what is counted when
+			 *   there is no sequence number to look it up by.
+			 */
+			void OnRtpPacketSent(std::optional<int64_t> sequenceNumber, size_t size, int64_t sentAtUs);
+
+			/**
+			 * Feed a received transport wide cc feedback, which is what the delay
+			 * based estimation is drawn from.
+			 */
+			void ReceiveTransportWideCcFeedback(
+			  const RTC::RTCP::FeedbackRtpTransportPacket* feedback, int64_t receivedAtUs);
+
+			/**
+			 * Feed what the RTCP Receiver Reports of a received RTCP packet measured,
+			 * added up over all the streams they report on, which is where the loss of
+			 * what we send is taken from.
+			 *
+			 * @remarks
+			 * - The loss may be negative, since a report gives back what a previous one
+			 *   was overcharged for.
+			 */
+			void ReceiveRtcpReceiverReport(int64_t lostPackets, int64_t expectedPackets);
+
+			/**
+			 * Feed a received REMB, which is what the remote endpoint says it is
+			 * willing to receive.
+			 */
+			void ReceiveEstimatedBitrate(int64_t bitrate);
+
+			/**
+			 * Feed the round trip time towards the remote endpoint.
+			 */
+			void OnRttUpdate(int64_t rttUs);
+
+			/**
+			 * Bound what may be estimated (bps). The bitrate to start from is not
+			 * among these, since it only applies before anything has been measured.
+			 */
+			void SetBitrateLimits(int64_t minBitrate, int64_t maxBitrate);
+
+			/**
+			 * Feed the total bitrate that the streams being sent would like to have
+			 * (bps), which is what tells whether there is any point in probing higher.
+			 */
+			void SetDesiredBitrate(int64_t desiredBitrate);
+
+			/**
+			 * Size of the overhead that every packet carries below RTP (bytes), which
+			 * the link bears along with the packets themselves.
+			 */
+			void SetPacketOverhead(size_t packetOverhead);
+
+			/**
+			 * Latest target bitrate (bps).
+			 */
+			int64_t GetAvailableBitrate() const;
+
+			/* Pure virtual methods inherited from ProbingScheduler::Listener. */
+		public:
+			void OnProbingSchedulerSendRtpPacket(
+			  ProbingScheduler* probingScheduler,
+			  RTC::RTP::Packet* packet,
+			  const Types::ProbeCluster& probeCluster) override;
+
+			/* Pure virtual methods inherited from RTC::TimerHandleInterface::Listener. */
+		public:
+			void OnTimer(TimerHandleInterface* timer) override;
+
+		private:
+			/**
+			 * Take note of a packet about to be sent and write its two extensions,
+			 * which is the same work for a packet of media and for one of a probe.
+			 *
+			 * @returns No value when the packet has no room for the transport wide
+			 *   sequence number, in which case nothing is taken note of.
+			 */
+			std::optional<int64_t> AddRtpPacket(
+			  RTC::RTP::Packet* packet,
+			  const RtpPacketToBeSentOptions& options,
+			  std::optional<Types::ProbeCluster> probeCluster);
+
+			/**
+			 * Run everything that depends on time having passed rather than on
+			 * anything having arrived, which is where the target may rise and where
+			 * probes are decided.
+			 */
+			void Process();
+
+			/**
+			 * Take note of the bursts that were asked for, which go out once the call
+			 * that asked for them is over.
+			 */
+			void AddProbeClusters(const std::vector<Types::ProbeClusterConfig>& clusterConfigs);
+
+			/**
+			 * Do everything the call leaves pending: hand the bursts to the scheduler
+			 * and tell the listener about the target, but only when something it
+			 * reasons about actually changed.
+			 *
+			 * @remarks
+			 * - It is the last thing done by every entry point that may leave something
+			 *   pending, and the only place where anything of this leaves the class.
+			 */
+			void ApplyPendingUpdate();
+
+			/**
+			 * Reconsider the target and take note of it when the listener has to be
+			 * told, which is what every path that may have moved it ends with.
+			 */
+			void MayNotifyTargetBitrate(int64_t nowUs);
+
+			/**
+			 * Bring the bounds into a range the pieces below accept and hand them
+			 * down.
+			 *
+			 * @param applyStartBitrate - Whether the bitrate to start from is handed
+			 *   down along with them. It only is the first time, since doing it again
+			 *   would force the target back to it and throw away everything the delay
+			 *   based path has learnt since.
+			 */
+			void ResetConstraints(int64_t nowUs, bool applyStartBitrate);
+
+		private:
+			// Passed by argument.
+			Listener* listener{ nullptr };
+			SharedInterface* shared{ nullptr };
+			// Allocated by this.
+			const std::unique_ptr<TimerHandleInterface> processTimer;
+			// Others.
+			SendPacketHistory sendPacketHistory;
+			FeedbackAdapter feedbackAdapter;
+			DelayBasedBwe delayBasedBwe;
+			RobustThroughputEstimator acknowledgedBitrateEstimator;
+			TargetRateController targetRateController;
+			AlrDetector alrDetector;
+			ProbeController probeController;
+			ProbeBitrateEstimator probeBitrateEstimator;
+			ProbingScheduler probingScheduler;
+			// What every packet carries below RTP (bytes), which is added to the size
+			// the rate control reasons about.
+			size_t packetOverhead{ 0 };
+			// Bounds in force, already brought into a range the pieces below accept.
+			// The one to start from is kept because a change of the other two has to
+			// be handed down along with it, and it is raised to the minimum where it
+			// is used rather than here, so that it stays what was asked for.
+			int64_t minBitrate{ 0 };
+			const int64_t startBitrate;
+			int64_t maxBitrate{ Types::BitrateInfinite };
+			// Whether anything has been sent and whether any transport-cc feedback has
+			// arrived, which tell apart a transport that is not being used from one
+			// whose remote endpoint does not report.
+			bool firstPacketSent{ false };
+			bool firstTransportFeedbackReceived{ false };
+			// Whether the sender was not filling the link when the latest feedback was
+			// handled, so that the instant it stopped being so can be told.
+			bool previouslyInAlr{ false };
+			// What the listener was last told about, which is what tells whether there
+			// is anything new to tell.
+			std::optional<int64_t> lastTargetBitrate;
+			uint8_t lastFractionLost{ 0 };
+			int64_t lastRttUs{ 0 };
+			LossBasedController::State lastLossBasedState{ LossBasedController::State::DELAY_BASED_ESTIMATE };
+			bool lastIsBandwidthLimited{ true };
+			// What the call being served leaves to be done once it is over.
+			PendingUpdate pendingUpdate;
+		};
+	} // namespace BWE
+} // namespace RTC
+
+#endif

package/worker/include/RTC/BWE/TargetRateController.hpp

@@ -62,6 +62,9 @@ namespace RTC
 				/**
 				 * Round trip time over which the bitrate is dropped regardless of loss.
 				 * A round trip time this long means the queues of the network are full.
+				 *
+				 * @remarks
+				 * - `Types::TimeUsInfinite` turns the backoff off.
 				 */
 				int64_t maxRttUs{ 3 * 1000 * 1000 };
 				/**
@@ -270,7 +273,7 @@ namespace RTC
 			// the target as soon as it has enough observations.
 			LossBasedController lossBasedController;
 			LossBasedController::State lossBasedState{ LossBasedController::State::DELAY_BASED_ESTIMATE };
-			struct RttBackoff rttBackoff;
+			RttBackoff rttBackoff;
 			// Lowest target of the last second, as a sliding window minimum of
 			// (instant in us, bitrate in bps) pairs.
 			std::deque<std::pair<int64_t, int64_t>> minBitrateHistory;

package/worker/include/RTC/BWE/TransportWideCcFeedbackGenerator.hpp

@@ -38,10 +38,11 @@ namespace RTC
 				/**
 				 * A feedback packet is ready for the remote sender.
 				 *
-				 * @param packet - Packet to send, only valid for the duration of this
-				 *   call.
+				 * @param packet - Packet to send. It belongs to the caller, so the
+				 *   listener must neither destroy it nor keep it around after this call
+				 *   returns.
 				 */
-				virtual void OnTransportWideCcFeedbackGeneratorSendRtcpPacket(
+				virtual void OnTransportWideCcFeedbackGeneratorSendPacket(
 				  TransportWideCcFeedbackGenerator* transportWideCcFeedbackGenerator,
 				  RTC::RTCP::FeedbackRtpTransportPacket* packet) = 0;
 			};
@@ -60,7 +61,7 @@ namespace RTC
 			 *
 			 * @param arrivalTimeUs - Instant the packet arrived.
 			 */
-			void IncomingPacket(int64_t arrivalTimeUs, const RTC::RTP::Packet* packet);
+			void ReceiveRtpPacket(int64_t arrivalTimeUs, const RTC::RTP::Packet* packet);
 
 #ifdef MS_TEST
 		public:

package/worker/include/RTC/BWE/TrendlineEstimator.hpp

@@ -103,8 +103,8 @@ namespace RTC
 			 * @param sendDeltaUs - Time elapsed between the send times of both groups.
 			 * @param arrivalDeltaUs - Time elapsed between the arrival times of both
 			 *   groups.
-			 * @param arrivalTimeUs - Arrival time of the latest group, in the remote
-			 *   clock reference.
+			 * @param arrivalTimeUs - Instant the latest group arrived at the remote
+			 *   endpoint, anchored to our own clock by the feedback adapter.
 			 */
 			void Update(int64_t sendDeltaUs, int64_t arrivalDeltaUs, int64_t arrivalTimeUs);
 
@@ -149,7 +149,7 @@ namespace RTC
 			double prevTrend{ 0 };
 			std::optional<double> timeOverUsingUs;
 			int overuseCounter{ 0 };
-			std::optional<int64_t> lastThresholdUpdateAtUs;
+			std::optional<int64_t> lastSampleArrivalTimeUs;
 			Types::BandwidthUsage state{ Types::BandwidthUsage::NORMAL };
 		};
 	} // namespace BWE

package/worker/include/RTC/Consumer.hpp

@@ -9,6 +9,7 @@
 #include "RTC/ConsumerTypes.hpp"
 #include "RTC/ProducerStreamManager.hpp"
 #include "RTC/RTCP/CompoundPacket.hpp"
+#include "RTC/RTCP/Feedback.hpp"
 #include "RTC/RTCP/FeedbackRtpNack.hpp"
 #include "RTC/RTCP/ReceiverReport.hpp"
 #include "RTC/RTP/HeaderExtensionIds.hpp"
@@ -153,7 +154,13 @@ namespace RTC
 		uint8_t GetWorstRemoteFractionLost(uint32_t mappedSsrc) const;
 		void ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket);
 		void ReceiveKeyFrameRequest(RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc);
-		void ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report, int64_t receivedAtUs);
+		/**
+		 * @returns What the link to the remote endpoint lost and what it was expected
+		 *   to deliver since the previous Receiver Report, or no value if this one
+		 *   measures no interval at all.
+		 */
+		std::optional<RTC::RTP::RtpStreamSend::Loss> ReceiveRtcpReceiverReport(
+		  RTC::RTCP::ReceiverReport* report, int64_t receivedAtUs);
 		void ReceiveRtcpXrReceiverReferenceTime(
 		  RTC::RTCP::ReceiverReferenceTime* report, int64_t receivedAtUs);
 		int64_t GetTransmissionRate(int64_t nowMs);

package/worker/include/RTC/RTP/RetransmissionBuffer.hpp

@@ -22,26 +22,47 @@ namespace RTC
 			{
 				void Reset();
 
-				// Original packet.
+				/**
+				 * Original packet.
+				 */
 				RTP::SharedPacket sharedPacket{ nullptr };
-				// Payload descriptor encoder.
+				/**
+				 * Payload descriptor encoder.
+				 */
 				std::unique_ptr<RTP::Codecs::PayloadDescriptor::Encoder> encoder{ nullptr };
-				// Correct SSRC since original packet may not have the same.
+				/**
+				 * Correct SSRC since original packet may not have the same.
+				 */
 				uint32_t ssrc{ 0 };
-				// Correct sequence number since original packet may not have the same.
+				/**
+				 * Correct sequence number since original packet may not have the same.
+				 */
 				uint16_t sequenceNumber{ 0 };
-				// Correct timestamp since original packet may not have the same.
+				/**
+				 * Correct timestamp since original packet may not have the same.
+				 */
 				uint32_t timestamp{ 0 };
-				// Correct marker bit since original packet may not have the same.
+				/**
+				 * Correct marker bit since original packet may not have the same.
+				 */
 				bool marker{ false };
-				// Last time this packet was resent.
+				/**
+				 * Time at which this packet was stored.
+				 */
+				int64_t storedAtMs{ 0 };
+				/**
+				 * Last time this packet was resent.
+				 */
 				int64_t resentAtMs{ 0 };
-				// Number of times this packet was resent.
+				/**
+				 * Number of times this packet was resent.
+				 */
 				uint8_t sentTimes{ 0 };
 			};
 
 		private:
-			static Item* FillItem(Item* item, RTP::Packet* packet, const RTP::SharedPacket& sharedPacket);
+			static Item* FillItem(
+			  Item* item, RTP::Packet* packet, const RTP::SharedPacket& sharedPacket, int64_t nowMs);
 
 		public:
 			RetransmissionBuffer(uint16_t maxItems, int64_t maxRetransmissionDelayMs, uint32_t clockRate);
@@ -49,8 +70,13 @@ namespace RTC
 
 			void Dump(int indentation = 0) const;
 			Item* Get(uint16_t seq) const;
-			bool Insert(RTP::Packet* packet, const RTP::SharedPacket& sharedPacket);
+			bool Insert(RTP::Packet* packet, const RTP::SharedPacket& sharedPacket, int64_t nowMs);
 			void Clear();
+			/**
+			 * Whether the given stored item was stored longer than the maximum
+			 * retransmission delay ago.
+			 */
+			bool IsTooOld(const Item* item, int64_t nowMs) const;
 
 		private:
 			Item* GetOldest() const;

package/worker/include/RTC/RTP/RtpStream.hpp

@@ -16,6 +16,22 @@ namespace RTC
 	{
 		class RtpStream
 		{
+		public:
+			/**
+			 * Largest jump forward in sequence number taken as a gap within the stream
+			 * rather than as a restart of it.
+			 */
+			static constexpr uint16_t MaxDropout{ 3000 };
+			/**
+			 * How far back a sequence number may land for its packet to be taken as
+			 * reordered rather than as a restart of the stream.
+			 */
+			static constexpr uint16_t MaxMisorder{ 1500 };
+			/**
+			 * Size of the RTP sequence number space, which is where it wraps around.
+			 */
+			static constexpr uint32_t RtpSeqMod{ 1 << 16 };
+
 		protected:
 			class Listener
 			{
@@ -171,6 +187,11 @@ namespace RTC
 				return this->fractionLost;
 			}
 
+			int32_t GetPacketsLost() const
+			{
+				return this->packetsLost;
+			}
+
 			float GetLossPercentage() const
 			{
 				return static_cast<float>(this->fractionLost) * 100 / 256;

package/worker/include/RTC/RTP/RtpStreamRecv.hpp

@@ -1,6 +1,7 @@
 #ifndef MS_RTC_RTP_RTP_STREAM_RECV_HPP
 #define MS_RTC_RTP_RTP_STREAM_RECV_HPP
 
+#include "common.hpp"
 #include "handles/TimerHandleInterface.hpp"
 #include "RTC/NackGenerator.hpp"
 #include "RTC/RTCP/Packet.hpp"