mediasoup 3.26.0 → 3.27.1

package/worker/include/RTC/BWE/AimdRateControl.hpp

@@ -0,0 +1,190 @@
+#ifndef MS_RTC_BWE_AIMD_RATE_CONTROL_HPP
+#define MS_RTC_BWE_AIMD_RATE_CONTROL_HPP
+
+#include "common.hpp"
+#include "RTC/BWE/BweTypes.hpp"
+#include "RTC/BWE/LinkCapacityEstimator.hpp"
+
+namespace RTC
+{
+	namespace BWE
+	{
+		/**
+		 * Decides the target bitrate out of what the delay based detector says about
+		 * the network.
+		 *
+		 * While no overuse is detected the bitrate is increased, multiplicatively
+		 * when the capacity of the link is still unknown and additively once it has
+		 * been estimated, since being close to the capacity makes a big step
+		 * expensive. On overuse it is decreased to slightly below the measured
+		 * throughput and held there until the queues of the network drain.
+		 */
+		class AimdRateControl
+		{
+		public:
+			struct AimdRateControlOptions
+			{
+				/**
+				 * Fraction of the measured throughput the bitrate is dropped to when
+				 * overuse is detected.
+				 */
+				double backoffFactor{ 0.85 };
+				/**
+				 * Whether the bitrate is not allowed to increase while the sender is in
+				 * an application limited region. In that region the network gives no
+				 * feedback about whether a higher bitrate would have worked, so an
+				 * increase cannot be validated.
+				 */
+				bool noBitrateIncreaseInAlr{ false };
+				/**
+				 * Whether the estimate of the link capacity is used as an upper bound.
+				 */
+				bool estimateBoundedIncrease{ true };
+				/**
+				 * Whether the current bitrate is taken as the lowest possible upper
+				 * bound, so that an estimate of the link capacity below it does not
+				 * drag the bitrate down.
+				 */
+				bool useCurrentEstimateAsMinUpperBound{ true };
+				/**
+				 * Whether this rate control runs in the endpoint that sends the media
+				 * whose capacity is being estimated, rather than in the one that
+				 * receives it. Only the sender can know that it's in an application
+				 * limited region, so `noBitrateIncreaseInAlr` has no effect otherwise.
+				 */
+				bool sendSide{ false };
+			};
+
+		private:
+			enum class RateControlState : uint8_t
+			{
+				HOLD,
+				INCREASE,
+				DECREASE
+			};
+
+		public:
+			AimdRateControl();
+
+			explicit AimdRateControl(AimdRateControlOptions options);
+
+			/**
+			 * Whether the target bitrate has been initialized, which happens either
+			 * when it's explicitly set or once a throughput has been measured for long
+			 * enough.
+			 */
+			bool IsValidEstimate() const
+			{
+				return this->bitrateIsInitialized;
+			}
+
+			/**
+			 * Current target bitrate (bps).
+			 */
+			int64_t GetLatestEstimate() const
+			{
+				return this->currentBitrate;
+			}
+
+			/**
+			 * Feed what the delay based detector says about the network.
+			 *
+			 * @param input - State of the network plus the acknowledged bitrate.
+			 * @param atTimeUs - Time at which the input was produced.
+			 *
+			 * @returns The new target bitrate (bps).
+			 */
+			int64_t Update(const Types::RateControlInput& input, int64_t atTimeUs);
+
+			/**
+			 * Set the target bitrate from outside the rate control, for instance out of
+			 * the bitrate measured by a probe.
+			 *
+			 * @param bitrate - New target bitrate (bps).
+			 */
+			void SetEstimate(int64_t bitrate, int64_t atTimeUs);
+
+			/**
+			 * @param startBitrate - Bitrate to start from (bps).
+			 */
+			void SetStartBitrate(int64_t startBitrate);
+
+			/**
+			 * @param minBitrate - Bitrate the target is never taken below (bps).
+			 */
+			void SetMinBitrate(int64_t minBitrate);
+
+			void SetRttUs(int64_t rttUs);
+
+			void SetInApplicationLimitedRegion(bool inAlr);
+
+			void SetNetworkStateEstimate(const std::optional<Types::NetworkStateEstimate>& estimate);
+
+			/**
+			 * Whether the bitrate can be reduced again, which requires that it has not
+			 * been changed for more than an RTT or that the throughput has fallen well
+			 * below the current target. It's what keeps a single overuse from
+			 * collapsing the bitrate in a few milliseconds.
+			 */
+			bool IsTimeToReduceFurther(int64_t atTimeUs, int64_t estimatedThroughput) const;
+
+			/**
+			 * As above, to be used when overusing before any throughput has been
+			 * measured.
+			 */
+			bool IsInitialTimeToReduceFurther(int64_t atTimeUs) const;
+
+			/**
+			 * Rate at which the bitrate is increased once it's close to the capacity of
+			 * the link (bps per second).
+			 */
+			double GetNearMaxIncreaseRateBpsPerSecond() const;
+
+			/**
+			 * How often feedback can be sent while devoting no more than 5% of the
+			 * current bitrate to it.
+			 */
+			int64_t GetFeedbackIntervalUs() const;
+
+		private:
+			void ChangeBitrate(const Types::RateControlInput& input, int64_t atTimeUs);
+
+			void ChangeState(const Types::RateControlInput& input, int64_t atTimeUs);
+
+			/**
+			 * Apply the configured bounds to a candidate bitrate.
+			 */
+			int64_t ClampBitrate(int64_t newBitrate) const;
+
+			/**
+			 * Increase used while the capacity of the link is unknown, which ramps up
+			 * fast to discover it.
+			 */
+			int64_t MultiplicativeRateIncrease(
+			  int64_t atTimeUs, std::optional<int64_t> lastTimeUs, int64_t currentBitrate) const;
+
+			/**
+			 * Increase used once the capacity of the link has been estimated, which
+			 * approaches it in small steps.
+			 */
+			int64_t AdditiveRateIncrease(int64_t atTimeUs, int64_t lastTimeUs) const;
+
+		private:
+			const AimdRateControlOptions options;
+			int64_t minConfiguredBitrate;
+			int64_t maxConfiguredBitrate;
+			int64_t currentBitrate;
+			int64_t latestEstimatedThroughput;
+			LinkCapacityEstimator linkCapacity;
+			std::optional<Types::NetworkStateEstimate> networkEstimate;
+			RateControlState rateControlState{ RateControlState::HOLD };
+			std::optional<int64_t> timeLastBitrateChangeUs;
+			std::optional<int64_t> timeFirstThroughputEstimateUs;
+			bool bitrateIsInitialized{ false };
+			bool inAlr{ false };
+			int64_t rttUs;
+		};
+	} // namespace BWE
+} // namespace RTC
+
+#endif

package/worker/include/RTC/BWE/BweTypes.hpp

@@ -0,0 +1,211 @@
+#ifndef MS_RTC_BWE_BWE_TYPES_HPP
+#define MS_RTC_BWE_BWE_TYPES_HPP
+
+#include "common.hpp"
+#include <limits>
+#include <string_view>
+#include <vector>
+
+namespace RTC
+{
+	namespace BWE
+	{
+		namespace Types
+		{
+			/**
+			 * Bitrate meaning that there is no limit at all (bps).
+			 */
+			constexpr int64_t BitrateInfinite{ std::numeric_limits<int64_t>::max() };
+
+			/**
+			 * Time meaning that no time was ever assigned.
+			 */
+			constexpr int64_t TimeUsInfinite{ std::numeric_limits<int64_t>::max() };
+
+			/**
+			 * How the network is behaving according to the delay based detector.
+			 */
+			enum class BandwidthUsage : uint8_t
+			{
+				/**
+				 * Queues are not growing.
+				 */
+				NORMAL,
+				/**
+				 * Queues are being emptied, so we are sending below the link capacity.
+				 */
+				UNDERUSING,
+				/**
+				 * Queues are filling up, so we are sending above the link capacity.
+				 */
+				OVERUSING
+			};
+
+			constexpr std::string_view bandwidthUsageToString(BandwidthUsage bandwidthUsage)
+			{
+				switch (bandwidthUsage)
+				{
+					case BandwidthUsage::NORMAL:
+					{
+						return "NORMAL";
+					}
+
+					case BandwidthUsage::UNDERUSING:
+					{
+						return "UNDERUSING";
+					}
+
+					case BandwidthUsage::OVERUSING:
+					{
+						return "OVERUSING";
+					}
+
+						NO_DEFAULT_GCC();
+				}
+			}
+
+			/**
+			 * Estimate of the capacity of the link, used to bound the rate control.
+			 */
+			struct NetworkStateEstimate
+			{
+				/**
+				 * Safe measure of the available capacity (bps).
+				 */
+				std::optional<int64_t> linkCapacityLower;
+				/**
+				 * Limit applied when increasing the bitrate (bps).
+				 */
+				std::optional<int64_t> linkCapacityUpper;
+			};
+
+			/**
+			 * Input given to the rate control on every update.
+			 */
+			struct RateControlInput
+			{
+				/**
+				 * How the network is behaving according to the delay based detector.
+				 */
+				BandwidthUsage bandwidthUsage;
+				/**
+				 * Bitrate acknowledged by the receiver (bps), or no value if it could
+				 * not be measured.
+				 */
+				std::optional<int64_t> estimatedThroughput;
+			};
+
+			/**
+			 * A packet that was sent and is being tracked until its feedback arrives.
+			 */
+			struct SentPacket
+			{
+				/**
+				 * Sequence number of the packet within the transport, unique over the
+				 * whole transport and increasing by one per tracked packet.
+				 *
+				 * @remarks
+				 * - It's independent of the feedback format. With transport-cc it
+				 *   matches the wide sequence number carried in the header extension,
+				 *   while with RFC 8888 the feedback identifies packets by SSRC and RTP
+				 *   sequence number and the adapter resolves them against this one.
+				 */
+				int64_t sequenceNumber;
+				/**
+				 * Time at which the packet was sent, or `Types::TimeUsInfinite` if it was
+				 * never assigned one.
+				 */
+				int64_t sendTimeUs{ TimeUsInfinite };
+				/**
+				 * Size of the packet including overhead up to the IP layer (bytes).
+				 */
+				size_t size{ 0 };
+				/**
+				 * Size of the preceding packets that are not part of any feedback
+				 * (bytes).
+				 */
+				size_t priorUnackedData{ 0 };
+				/**
+				 * Data that was in flight when the packet was sent, itself included and
+				 * the data that no feedback covers excluded (bytes).
+				 */
+				size_t dataInFlight{ 0 };
+				/**
+				 * Whether it's an audio packet. False for video, padding and RTX.
+				 */
+				bool audio{ false };
+			};
+
+			/**
+			 * What a feedback reported about a single sent packet.
+			 */
+			struct PacketResult
+			{
+				/**
+				 * Orders packets by the time at which they arrived.
+				 *
+				 * @remarks
+				 * - Send time and sequence number break the ties, which are common
+				 *   because the arrival times within a burst are reported with a coarse
+				 *   resolution.
+				 */
+				struct ReceiveTimeOrder
+				{
+					bool operator()(const PacketResult& lhs, const PacketResult& rhs) const;
+				};
+
+				/**
+				 * Whether the packet reached the receiver at all.
+				 */
+				bool IsReceived() const
+				{
+					return this->receiveTimeUs.has_value();
+				}
+
+				SentPacket sentPacket;
+				/**
+				 * Time at which the packet arrived, in the remote clock reference, or no
+				 * value if it was reported as lost.
+				 */
+				std::optional<int64_t> receiveTimeUs;
+			};
+
+			/**
+			 * A whole feedback message, reporting on the packets sent so far.
+			 */
+			struct TransportPacketsFeedback
+			{
+				/**
+				 * The packets that reached the receiver, ordered by arrival time.
+				 *
+				 * @remarks
+				 * - Send time and sequence number break the ties, which are common
+				 *   because the arrival times within a burst are reported with a coarse
+				 *   resolution.
+				 */
+				std::vector<PacketResult> SortedByReceiveTime() const;
+
+				/**
+				 * The packets that reached the receiver, in the order they were
+				 * reported.
+				 */
+				std::vector<PacketResult> ReceivedWithSendInfo() const;
+
+				/**
+				 * Time at which this feedback was received, in our own clock reference.
+				 */
+				int64_t feedbackTimeUs;
+				/**
+				 * Data that was sent and no feedback has reported on yet (bytes).
+				 */
+				size_t dataInFlight{ 0 };
+				/**
+				 * Every packet the feedback reports on, received and lost alike.
+				 */
+				std::vector<PacketResult> packetFeedbacks;
+			};
+		} // namespace Types
+	} // namespace BWE
+} // namespace RTC
+
+#endif

package/worker/include/RTC/BWE/DelayBasedBwe.hpp

@@ -0,0 +1,173 @@
+#ifndef MS_RTC_BWE_DELAY_BASED_BWE_HPP
+#define MS_RTC_BWE_DELAY_BASED_BWE_HPP
+
+#include "common.hpp"
+#include "RTC/BWE/AimdRateControl.hpp"
+#include "RTC/BWE/BweTypes.hpp"
+#include "RTC/BWE/InterArrivalDelta.hpp"
+#include "RTC/BWE/TrendlineEstimator.hpp"
+
+namespace RTC
+{
+	namespace BWE
+	{
+		/**
+		 * Estimates the available bitrate out of the delay between the packets that
+		 * were sent and the feedback reporting when they arrived.
+		 *
+		 * It's the glue of the delay based path: the packets of each feedback are
+		 * grouped into send bursts, the delay between consecutive groups feeds the
+		 * trendline detector, and what the detector says about the network drives
+		 * the rate control.
+		 */
+		class DelayBasedBwe
+		{
+		public:
+			struct DelayBasedBweOptions
+			{
+				/**
+				 * Whether audio and video get a delay detector of their own instead of
+				 * sharing one.
+				 */
+				bool separateAudioPackets{ false };
+				/**
+				 * Number of audio packets without a video one after which the audio
+				 * detector becomes the active one.
+				 *
+				 * @remarks
+				 * - Only used when `separateAudioPackets` is set.
+				 */
+				int64_t separateAudioPacketThreshold{ 10 };
+				/**
+				 * Time without a video packet after which the audio detector becomes
+				 * the active one.
+				 *
+				 * @remarks
+				 * - Only used when `separateAudioPackets` is set.
+				 */
+				int64_t separateAudioTimeThresholdUs{ 1000 * 1000 };
+			};
+
+			struct Result
+			{
+				/**
+				 * Whether the target bitrate was recomputed.
+				 */
+				bool updated{ false };
+				/**
+				 * Whether the target bitrate comes from the bitrate measured by a
+				 * probe.
+				 */
+				bool probe{ false };
+				/**
+				 * Target bitrate (bps).
+				 */
+				int64_t targetBitrate{ 0 };
+				/**
+				 * Whether the detector went back to normal after having been
+				 * underusing.
+				 */
+				bool recoveredFromOveruse{ false };
+				/**
+				 * What the delay based detector says about the network.
+				 */
+				Types::BandwidthUsage delayDetectorState{ Types::BandwidthUsage::NORMAL };
+			};
+
+		public:
+			DelayBasedBwe();
+
+			explicit DelayBasedBwe(DelayBasedBweOptions options);
+
+			/**
+			 * Feed a whole feedback message.
+			 *
+			 * @param feedback - What the receiver reported about the packets sent so
+			 *   far.
+			 * @param ackedBitrate - Bitrate acknowledged by the receiver (bps), or no
+			 *   value if it could not be measured.
+			 * @param probeBitrate - Bitrate measured by a probe (bps), or no value if
+			 *   no probe was completed.
+			 * @param networkEstimate - Estimate of the capacity of the link, used to
+			 *   bound the rate control.
+			 * @param inAlr - Whether the sender is in an application limited region.
+			 */
+			Result IncomingPacketFeedbackVector(
+			  const Types::TransportPacketsFeedback& feedback,
+			  std::optional<int64_t> ackedBitrate,
+			  std::optional<int64_t> probeBitrate,
+			  const std::optional<Types::NetworkStateEstimate>& networkEstimate,
+			  bool inAlr);
+
+			void OnRttUpdate(int64_t avgRttUs);
+
+			/**
+			 * @param startBitrate - Bitrate to start from (bps).
+			 */
+			void SetStartBitrate(int64_t startBitrate);
+
+			/**
+			 * @param minBitrate - Bitrate the target is never taken below (bps).
+			 */
+			void SetMinBitrate(int64_t minBitrate);
+
+			/**
+			 * Current target bitrate of the rate control (bps), or no value if there
+			 * is no valid estimate yet.
+			 *
+			 * @remarks
+			 * - Unlike `GetLastEstimate()`, this is what the rate control holds right
+			 *   now rather than the latest value that was reported.
+			 */
+			std::optional<int64_t> GetLatestEstimate() const;
+
+			/**
+			 * Latest target bitrate that was reported (bps).
+			 */
+			int64_t GetLastEstimate() const
+			{
+				return this->prevBitrate;
+			}
+
+			/**
+			 * Latest state of the delay based detector that was reported.
+			 */
+			Types::BandwidthUsage GetLastState() const
+			{
+				return this->prevState;
+			}
+
+		private:
+			void IncomingPacketFeedback(const Types::PacketResult& packetResult, int64_t atTimeUs);
+
+			Result MaybeUpdateEstimate(
+			  std::optional<int64_t> ackedBitrate,
+			  std::optional<int64_t> probeBitrate,
+			  bool recoveredFromOveruse,
+			  int64_t atTimeUs);
+
+			/**
+			 * Recompute the target bitrate.
+			 *
+			 * @returns Whether there is a valid estimate.
+			 */
+			bool UpdateEstimate(int64_t atTimeUs, std::optional<int64_t> ackedBitrate, int64_t& targetBitrate);
+
+		private:
+			const DelayBasedBweOptions options;
+			int64_t audioPacketsSinceLastVideo{ 0 };
+			std::optional<int64_t> lastVideoPacketRecvTimeUs;
+			std::optional<InterArrivalDelta> videoInterArrivalDelta;
+			std::optional<InterArrivalDelta> audioInterArrivalDelta;
+			std::optional<TrendlineEstimator> videoDelayDetector;
+			std::optional<TrendlineEstimator> audioDelayDetector;
+			TrendlineEstimator* activeDelayDetector{ nullptr };
+			std::optional<int64_t> lastSeenPacketUs;
+			AimdRateControl rateControl;
+			int64_t prevBitrate{ 0 };
+			Types::BandwidthUsage prevState{ Types::BandwidthUsage::NORMAL };
+		};
+	} // namespace BWE
+} // namespace RTC
+
+#endif

package/worker/include/RTC/BWE/FeedbackAdapter.hpp

@@ -0,0 +1,72 @@
+#ifndef MS_RTC_BWE_FEEDBACK_ADAPTER_HPP
+#define MS_RTC_BWE_FEEDBACK_ADAPTER_HPP
+
+#include "common.hpp"
+#include "RTC/BWE/BweTypes.hpp"
+#include "RTC/BWE/SendPacketHistory.hpp"
+#include "RTC/RTCP/FeedbackRtpTransport.hpp"
+
+namespace RTC
+{
+	namespace BWE
+	{
+		/**
+		 * Turns a feedback message into what it reports about the packets that were
+		 * sent, resolved against the history of sent packets.
+		 *
+		 * It's the only place that knows what a feedback looks like, so that
+		 * supporting another format is a matter of adding another method here.
+		 *
+		 * The arrival times a feedback carries come from the remote clock, against
+		 * a base time that wraps around, so they are unusable as they are. The
+		 * first feedback is anchored on the time it was received and every later
+		 * one advances that anchor by the elapsed base time, which puts the arrival
+		 * times of all of them on a single timeline. Only their differences matter
+		 * downstream.
+		 *
+		 * @see https://datatracker.ietf.org/doc/html/draft-holmer-rmcat-transport-wide-cc-extensions
+		 * @see https://datatracker.ietf.org/doc/html/rfc8888
+		 */
+		class FeedbackAdapter
+		{
+		public:
+			explicit FeedbackAdapter(SendPacketHistory* sendPacketHistory);
+
+			/**
+			 * Process a transport-cc feedback.
+			 *
+			 * @param receivedAtUs - Time at which the feedback was received.
+			 * @returns What the feedback reports about the packets that were sent, in
+			 *   the order it reports them, or no value if it reports on no packet the
+			 *   history still holds.
+			 */
+			std::optional<Types::TransportPacketsFeedback> ProcessTransportFeedback(
+			  const RTCP::FeedbackRtpTransportPacket* feedback, int64_t receivedAtUs);
+
+		private:
+			/**
+			 * Recover the sequence number the history gave to a packet out of the
+			 * lowest bits of it that the wire carries.
+			 *
+			 * @param previousSequenceNumber - Sequence number the previous packet the
+			 *   feedback reported on resolved into, which the returned one is the
+			 *   closest to.
+			 */
+			static int64_t UnwrapSequenceNumber(uint16_t wideSequenceNumber, int64_t previousSequenceNumber);
+
+		private:
+			SendPacketHistory* const sendPacketHistory;
+			// Our own time the arrival times of the current feedback are given
+			// against.
+			int64_t currentOffsetUs{ 0 };
+			// Base time of the latest feedback processed, in the remote clock, or no
+			// value if no feedback was ever processed.
+			std::optional<int64_t> lastFeedbackBaseTimeUs;
+			// Sequence number the latest packet reported on resolved into, or no
+			// value if no feedback was ever processed.
+			std::optional<int64_t> lastUnwrappedSequenceNumber;
+		};
+	} // namespace BWE
+} // namespace RTC
+
+#endif