mediasoup 3.26.0 → 3.27.0

package/worker/include/RTC/BWE/BweTypes.hpp

@@ -0,0 +1,211 @@
+#ifndef MS_RTC_BWE_BWE_TYPES_HPP
+#define MS_RTC_BWE_BWE_TYPES_HPP
+
+#include "common.hpp"
+#include <limits>
+#include <string_view>
+#include <vector>
+
+namespace RTC
+{
+	namespace BWE
+	{
+		namespace Types
+		{
+			/**
+			 * Bitrate meaning that there is no limit at all (bps).
+			 */
+			constexpr int64_t BitrateInfinite{ std::numeric_limits<int64_t>::max() };
+
+			/**
+			 * Time meaning that no time was ever assigned.
+			 */
+			constexpr int64_t TimeUsInfinite{ std::numeric_limits<int64_t>::max() };
+
+			/**
+			 * How the network is behaving according to the delay based detector.
+			 */
+			enum class BandwidthUsage : uint8_t
+			{
+				/**
+				 * Queues are not growing.
+				 */
+				NORMAL,
+				/**
+				 * Queues are being emptied, so we are sending below the link capacity.
+				 */
+				UNDERUSING,
+				/**
+				 * Queues are filling up, so we are sending above the link capacity.
+				 */
+				OVERUSING
+			};
+
+			constexpr std::string_view BandwidthUsageToString(BandwidthUsage bandwidthUsage)
+			{
+				switch (bandwidthUsage)
+				{
+					case BandwidthUsage::NORMAL:
+					{
+						return "NORMAL";
+					}
+
+					case BandwidthUsage::UNDERUSING:
+					{
+						return "UNDERUSING";
+					}
+
+					case BandwidthUsage::OVERUSING:
+					{
+						return "OVERUSING";
+					}
+
+						NO_DEFAULT_GCC();
+				}
+			}
+
+			/**
+			 * Estimate of the capacity of the link, used to bound the rate control.
+			 */
+			struct NetworkStateEstimate
+			{
+				/**
+				 * Safe measure of the available capacity (bps).
+				 */
+				std::optional<int64_t> linkCapacityLower;
+				/**
+				 * Limit applied when increasing the bitrate (bps).
+				 */
+				std::optional<int64_t> linkCapacityUpper;
+			};
+
+			/**
+			 * Input given to the rate control on every update.
+			 */
+			struct RateControlInput
+			{
+				/**
+				 * How the network is behaving according to the delay based detector.
+				 */
+				BandwidthUsage bandwidthUsage;
+				/**
+				 * Bitrate acknowledged by the receiver (bps), or no value if it could
+				 * not be measured.
+				 */
+				std::optional<int64_t> estimatedThroughput;
+			};
+
+			/**
+			 * A packet that was sent and is being tracked until its feedback arrives.
+			 */
+			struct SentPacket
+			{
+				/**
+				 * Sequence number of the packet within the transport, unique over the
+				 * whole transport and increasing by one per tracked packet.
+				 *
+				 * @remarks
+				 * - It's independent of the feedback format. With transport-cc it
+				 *   matches the wide sequence number carried in the header extension,
+				 *   while with RFC 8888 the feedback identifies packets by SSRC and RTP
+				 *   sequence number and the adapter resolves them against this one.
+				 */
+				int64_t sequenceNumber;
+				/**
+				 * Time at which the packet was sent, or `Types::TimeUsInfinite` if it was
+				 * never assigned one.
+				 */
+				int64_t sendTimeUs{ TimeUsInfinite };
+				/**
+				 * Size of the packet including overhead up to the IP layer (bytes).
+				 */
+				size_t size{ 0 };
+				/**
+				 * Size of the preceding packets that are not part of any feedback
+				 * (bytes).
+				 */
+				size_t priorUnackedData{ 0 };
+				/**
+				 * Data that was in flight when the packet was sent, itself included and
+				 * the data that no feedback covers excluded (bytes).
+				 */
+				size_t dataInFlight{ 0 };
+				/**
+				 * Whether it's an audio packet. False for video, padding and RTX.
+				 */
+				bool audio{ false };
+			};
+
+			/**
+			 * What a feedback reported about a single sent packet.
+			 */
+			struct PacketResult
+			{
+				/**
+				 * Orders packets by the time at which they arrived.
+				 *
+				 * @remarks
+				 * - Send time and sequence number break the ties, which are common
+				 *   because the arrival times within a burst are reported with a coarse
+				 *   resolution.
+				 */
+				struct ReceiveTimeOrder
+				{
+					bool operator()(const PacketResult& lhs, const PacketResult& rhs) const;
+				};
+
+				/**
+				 * Whether the packet reached the receiver at all.
+				 */
+				bool IsReceived() const
+				{
+					return this->receiveTimeUs.has_value();
+				}
+
+				SentPacket sentPacket;
+				/**
+				 * Time at which the packet arrived, in the remote clock reference, or no
+				 * value if it was reported as lost.
+				 */
+				std::optional<int64_t> receiveTimeUs;
+			};
+
+			/**
+			 * A whole feedback message, reporting on the packets sent so far.
+			 */
+			struct TransportPacketsFeedback
+			{
+				/**
+				 * The packets that reached the receiver, ordered by arrival time.
+				 *
+				 * @remarks
+				 * - Send time and sequence number break the ties, which are common
+				 *   because the arrival times within a burst are reported with a coarse
+				 *   resolution.
+				 */
+				std::vector<PacketResult> SortedByReceiveTime() const;
+
+				/**
+				 * The packets that reached the receiver, in the order they were
+				 * reported.
+				 */
+				std::vector<PacketResult> ReceivedWithSendInfo() const;
+
+				/**
+				 * Time at which this feedback was received, in our own clock reference.
+				 */
+				int64_t feedbackTimeUs;
+				/**
+				 * Data that was sent and no feedback has reported on yet (bytes).
+				 */
+				size_t dataInFlight{ 0 };
+				/**
+				 * Every packet the feedback reports on, received and lost alike.
+				 */
+				std::vector<PacketResult> packetFeedbacks;
+			};
+		} // namespace Types
+	} // namespace BWE
+} // namespace RTC
+
+#endif

package/worker/include/RTC/BWE/DelayBasedBwe.hpp

@@ -0,0 +1,173 @@
+#ifndef MS_RTC_BWE_DELAY_BASED_BWE_HPP
+#define MS_RTC_BWE_DELAY_BASED_BWE_HPP
+
+#include "common.hpp"
+#include "RTC/BWE/AimdRateControl.hpp"
+#include "RTC/BWE/BweTypes.hpp"
+#include "RTC/BWE/InterArrivalDelta.hpp"
+#include "RTC/BWE/TrendlineEstimator.hpp"
+
+namespace RTC
+{
+	namespace BWE
+	{
+		/**
+		 * Estimates the available bitrate out of the delay between the packets that
+		 * were sent and the feedback reporting when they arrived.
+		 *
+		 * It's the glue of the delay based path: the packets of each feedback are
+		 * grouped into send bursts, the delay between consecutive groups feeds the
+		 * trendline detector, and what the detector says about the network drives
+		 * the rate control.
+		 */
+		class DelayBasedBwe
+		{
+		public:
+			struct DelayBasedBweOptions
+			{
+				/**
+				 * Whether audio and video get a delay detector of their own instead of
+				 * sharing one.
+				 */
+				bool separateAudioPackets{ false };
+				/**
+				 * Number of audio packets without a video one after which the audio
+				 * detector becomes the active one.
+				 *
+				 * @remarks
+				 * - Only used when `separateAudioPackets` is set.
+				 */
+				int64_t separateAudioPacketThreshold{ 10 };
+				/**
+				 * Time without a video packet after which the audio detector becomes
+				 * the active one.
+				 *
+				 * @remarks
+				 * - Only used when `separateAudioPackets` is set.
+				 */
+				int64_t separateAudioTimeThresholdUs{ 1000 * 1000 };
+			};
+
+			struct Result
+			{
+				/**
+				 * Whether the target bitrate was recomputed.
+				 */
+				bool updated{ false };
+				/**
+				 * Whether the target bitrate comes from the bitrate measured by a
+				 * probe.
+				 */
+				bool probe{ false };
+				/**
+				 * Target bitrate (bps).
+				 */
+				int64_t targetBitrate{ 0 };
+				/**
+				 * Whether the detector went back to normal after having been
+				 * underusing.
+				 */
+				bool recoveredFromOveruse{ false };
+				/**
+				 * What the delay based detector says about the network.
+				 */
+				Types::BandwidthUsage delayDetectorState{ Types::BandwidthUsage::NORMAL };
+			};
+
+		public:
+			DelayBasedBwe();
+
+			explicit DelayBasedBwe(DelayBasedBweOptions options);
+
+			/**
+			 * Feed a whole feedback message.
+			 *
+			 * @param feedback - What the receiver reported about the packets sent so
+			 *   far.
+			 * @param ackedBitrate - Bitrate acknowledged by the receiver (bps), or no
+			 *   value if it could not be measured.
+			 * @param probeBitrate - Bitrate measured by a probe (bps), or no value if
+			 *   no probe was completed.
+			 * @param networkEstimate - Estimate of the capacity of the link, used to
+			 *   bound the rate control.
+			 * @param inAlr - Whether the sender is in an application limited region.
+			 */
+			Result IncomingPacketFeedbackVector(
+			  const Types::TransportPacketsFeedback& feedback,
+			  std::optional<int64_t> ackedBitrate,
+			  std::optional<int64_t> probeBitrate,
+			  const std::optional<Types::NetworkStateEstimate>& networkEstimate,
+			  bool inAlr);
+
+			void OnRttUpdate(int64_t avgRttUs);
+
+			/**
+			 * @param startBitrate - Bitrate to start from (bps).
+			 */
+			void SetStartBitrate(int64_t startBitrate);
+
+			/**
+			 * @param minBitrate - Bitrate the target is never taken below (bps).
+			 */
+			void SetMinBitrate(int64_t minBitrate);
+
+			/**
+			 * Current target bitrate of the rate control (bps), or no value if there
+			 * is no valid estimate yet.
+			 *
+			 * @remarks
+			 * - Unlike `GetLastEstimate()`, this is what the rate control holds right
+			 *   now rather than the latest value that was reported.
+			 */
+			std::optional<int64_t> GetLatestEstimate() const;
+
+			/**
+			 * Latest target bitrate that was reported (bps).
+			 */
+			int64_t GetLastEstimate() const
+			{
+				return this->prevBitrate;
+			}
+
+			/**
+			 * Latest state of the delay based detector that was reported.
+			 */
+			Types::BandwidthUsage GetLastState() const
+			{
+				return this->prevState;
+			}
+
+		private:
+			void IncomingPacketFeedback(const Types::PacketResult& packetResult, int64_t atTimeUs);
+
+			Result MaybeUpdateEstimate(
+			  std::optional<int64_t> ackedBitrate,
+			  std::optional<int64_t> probeBitrate,
+			  bool recoveredFromOveruse,
+			  int64_t atTimeUs);
+
+			/**
+			 * Recompute the target bitrate.
+			 *
+			 * @returns Whether there is a valid estimate.
+			 */
+			bool UpdateEstimate(int64_t atTimeUs, std::optional<int64_t> ackedBitrate, int64_t& targetBitrate);
+
+		private:
+			const DelayBasedBweOptions options;
+			int64_t audioPacketsSinceLastVideo{ 0 };
+			std::optional<int64_t> lastVideoPacketRecvTimeUs;
+			std::optional<InterArrivalDelta> videoInterArrivalDelta;
+			std::optional<InterArrivalDelta> audioInterArrivalDelta;
+			std::optional<TrendlineEstimator> videoDelayDetector;
+			std::optional<TrendlineEstimator> audioDelayDetector;
+			TrendlineEstimator* activeDelayDetector{ nullptr };
+			std::optional<int64_t> lastSeenPacketUs;
+			AimdRateControl rateControl;
+			int64_t prevBitrate{ 0 };
+			Types::BandwidthUsage prevState{ Types::BandwidthUsage::NORMAL };
+		};
+	} // namespace BWE
+} // namespace RTC
+
+#endif

package/worker/include/RTC/BWE/FeedbackAdapter.hpp

@@ -0,0 +1,72 @@
+#ifndef MS_RTC_BWE_FEEDBACK_ADAPTER_HPP
+#define MS_RTC_BWE_FEEDBACK_ADAPTER_HPP
+
+#include "common.hpp"
+#include "RTC/BWE/BweTypes.hpp"
+#include "RTC/BWE/SendPacketHistory.hpp"
+#include "RTC/RTCP/FeedbackRtpTransport.hpp"
+
+namespace RTC
+{
+	namespace BWE
+	{
+		/**
+		 * Turns a feedback message into what it reports about the packets that were
+		 * sent, resolved against the history of sent packets.
+		 *
+		 * It's the only place that knows what a feedback looks like, so that
+		 * supporting another format is a matter of adding another method here.
+		 *
+		 * The arrival times a feedback carries come from the remote clock, against
+		 * a base time that wraps around, so they are unusable as they are. The
+		 * first feedback is anchored on the time it was received and every later
+		 * one advances that anchor by the elapsed base time, which puts the arrival
+		 * times of all of them on a single timeline. Only their differences matter
+		 * downstream.
+		 *
+		 * @see https://datatracker.ietf.org/doc/html/draft-holmer-rmcat-transport-wide-cc-extensions
+		 * @see https://datatracker.ietf.org/doc/html/rfc8888
+		 */
+		class FeedbackAdapter
+		{
+		public:
+			explicit FeedbackAdapter(SendPacketHistory* sendPacketHistory);
+
+			/**
+			 * Process a transport-cc feedback.
+			 *
+			 * @param receivedAtUs - Time at which the feedback was received.
+			 * @returns What the feedback reports about the packets that were sent, in
+			 *   the order it reports them, or no value if it reports on no packet the
+			 *   history still holds.
+			 */
+			std::optional<Types::TransportPacketsFeedback> ProcessTransportFeedback(
+			  const RTCP::FeedbackRtpTransportPacket* feedback, int64_t receivedAtUs);
+
+		private:
+			/**
+			 * Recover the sequence number the history gave to a packet out of the
+			 * lowest bits of it that the wire carries.
+			 *
+			 * @param previousSequenceNumber - Sequence number the previous packet the
+			 *   feedback reported on resolved into, which the returned one is the
+			 *   closest to.
+			 */
+			static int64_t UnwrapSequenceNumber(uint16_t wideSequenceNumber, int64_t previousSequenceNumber);
+
+		private:
+			SendPacketHistory* const sendPacketHistory;
+			// Our own time the arrival times of the current feedback are given
+			// against.
+			int64_t currentOffsetUs{ 0 };
+			// Base time of the latest feedback processed, in the remote clock, or no
+			// value if no feedback was ever processed.
+			std::optional<int64_t> lastFeedbackBaseTimeUs;
+			// Sequence number the latest packet reported on resolved into, or no
+			// value if no feedback was ever processed.
+			std::optional<int64_t> lastUnwrappedSequenceNumber;
+		};
+	} // namespace BWE
+} // namespace RTC
+
+#endif

package/worker/include/RTC/BWE/InterArrivalDelta.hpp

@@ -0,0 +1,106 @@
+#ifndef MS_RTC_BWE_INTER_ARRIVAL_DELTA_HPP
+#define MS_RTC_BWE_INTER_ARRIVAL_DELTA_HPP
+
+#include "common.hpp"
+
+namespace RTC
+{
+	namespace BWE
+	{
+		/**
+		 * Groups sent packets into send bursts and computes the deltas between two
+		 * consecutive bursts.
+		 *
+		 * All packets sent within `sendTimeGroupLengthUs` of the first packet of a
+		 * group belong to that group. This means that a burst of packets written
+		 * into the socket at once produces a single sample rather than one per
+		 * packet, which is what prevents a key frame being forwarded as a burst from
+		 * being read as network congestion.
+		 *
+		 * The computed deltas are the input of the trendline estimator: if the
+		 * arrival delta grows faster than the send delta then queues are building up
+		 * somewhere in the network.
+		 */
+		class InterArrivalDelta
+		{
+		public:
+			/**
+			 * Deltas between two consecutive groups of packets sent in a burst.
+			 */
+			struct Deltas
+			{
+				/**
+				 * Time elapsed between the send times of both groups (us).
+				 */
+				int64_t sendDeltaUs;
+
+				/**
+				 * Time elapsed between the arrival times of both groups (us).
+				 */
+				int64_t arrivalDeltaUs;
+
+				/**
+				 * Difference of size between both groups (bytes).
+				 */
+				int64_t sizeDelta;
+			};
+
+		private:
+			struct SendTimeGroup
+			{
+				bool IsFirstPacket() const
+				{
+					return !this->completeTimeUs.has_value();
+				}
+
+				size_t size{ 0 };
+				int64_t firstSendTimeUs{ 0 };
+				int64_t sendTimeUs{ 0 };
+				int64_t firstArrivalTimeUs{ 0 };
+				std::optional<int64_t> completeTimeUs;
+				int64_t lastFeedbackAtUs{ 0 };
+			};
+
+		public:
+			/**
+			 * @param sendTimeGroupLengthUs - Every packet sent within this time of the
+			 *   first packet of a group belongs to that group.
+			 */
+			explicit InterArrivalDelta(int64_t sendTimeGroupLengthUs);
+
+			/**
+			 * Feed a sent packet whose arrival time is already known.
+			 *
+			 * @param sendTimeUs - Time at which the packet was sent.
+			 * @param arrivalTimeUs - Time at which the packet arrived, in the remote
+			 *   clock reference.
+			 * @param feedbackAtUs - Time, in our own clock reference, at which the
+			 *   RTCP feedback reporting this packet was received. It's the very same
+			 *   value for every packet within a given feedback, and it's used to
+			 *   detect that the remote clock has jumped by comparing how much it
+			 *   advanced against how much ours did.
+			 * @param packetSize - Size of the packet (bytes).
+			 *
+			 * @returns The deltas between the two latest complete groups, or no value
+			 *   if they cannot be computed yet.
+			 */
+			std::optional<Deltas> ComputeDeltas(
+			  int64_t sendTimeUs, int64_t arrivalTimeUs, int64_t feedbackAtUs, size_t packetSize);
+
+		private:
+			bool IsNewSendTimeGroup(int64_t arrivalTimeUs, int64_t sendTimeUs) const;
+
+			bool BelongsToBurst(int64_t arrivalTimeUs, int64_t sendTimeUs) const;
+
+			void Reset();
+
+		private:
+			const int64_t sendTimeGroupLengthUs;
+			SendTimeGroup currentGroup;
+			SendTimeGroup prevGroup;
+			size_t numConsecutiveReorderedGroups{ 0 };
+		};
+	} // namespace BWE
+} // namespace RTC
+
+#endif

package/worker/include/RTC/BWE/LinkCapacityEstimator.hpp

@@ -0,0 +1,84 @@
+#ifndef MS_RTC_BWE_LINK_CAPACITY_ESTIMATOR_HPP
+#define MS_RTC_BWE_LINK_CAPACITY_ESTIMATOR_HPP
+
+#include "common.hpp"
+
+namespace RTC
+{
+	namespace BWE
+	{
+		/**
+		 * Estimates the capacity of the link out of the bitrates at which
+		 * congestion was observed.
+		 *
+		 * Every sample is fed into an exponential filter, and the variance of the
+		 * estimate is tracked along with it so that a confidence interval can be
+		 * derived. The rate control uses that interval to tell whether the measured
+		 * throughput still has anything to do with the capacity it had estimated: a
+		 * throughput outside of it means that the link changed, so the estimate is
+		 * dropped instead of being slowly filtered towards the new reality.
+		 */
+		class LinkCapacityEstimator
+		{
+		public:
+			/**
+			 * Feed the bitrate at which the network started to be overused, which is
+			 * an observation of where the capacity of the link is.
+			 *
+			 * @param ackedBitrate - Acknowledged bitrate when overuse was detected
+			 *   (bps).
+			 */
+			void OnOveruseDetected(int64_t ackedBitrate);
+
+			/**
+			 * Feed the bitrate measured by a probe. It's given far more weight than an
+			 * overuse, since a probe measures the capacity of the link directly rather
+			 * than inferring it from the delay.
+			 *
+			 * @param probeBitrate - Bitrate measured by the probe (bps).
+			 */
+			void OnProbeRate(int64_t probeBitrate);
+
+			/**
+			 * Current estimate of the capacity of the link (bps), or no value if
+			 * nothing has been observed yet.
+			 */
+			std::optional<int64_t> GetEstimate() const;
+
+			/**
+			 * Upper end of the confidence interval of the estimate (bps), or
+			 * `Types::BitrateInfinite` if there is no estimate yet.
+			 */
+			int64_t GetUpperBound() const;
+
+			/**
+			 * Lower end of the confidence interval of the estimate (bps), or 0 if
+			 * there is no estimate yet.
+			 */
+			int64_t GetLowerBound() const;
+
+			/**
+			 * Drop the estimate, so that the next sample is taken as is instead of
+			 * being filtered against a value that no longer holds.
+			 */
+			void Reset();
+
+		private:
+			void Update(int64_t bitrate, double alpha);
+
+			/**
+			 * Standard deviation of the estimate, derived from the normalized variance
+			 * being tracked. Only meaningful once there is an estimate.
+			 */
+			double GetDeviationEstimateKbps() const;
+
+		private:
+			// NOTE: The filter works in kbps as double since the bounds applied to the
+			// variance are calibrated in those units.
+			std::optional<double> estimateKbps;
+			double deviationKbps{ 0.4 };
+		};
+	} // namespace BWE
+} // namespace RTC
+
+#endif