mediasoup 3.26.0 → 3.27.0

package/worker/src/RTC/BWE/DelayBasedBwe.cpp

@@ -0,0 +1,275 @@
+#define MS_CLASS "RTC::BWE::DelayBasedBwe"
+// #define MS_LOG_DEV_LEVEL 3
+
+#include "RTC/BWE/DelayBasedBwe.hpp"
+#include "Logger.hpp"
+
+namespace RTC
+{
+	namespace BWE
+	{
+		/* Static. */
+
+		// Time without any packet after which everything learnt about the stream is
+		// dropped.
+		static constexpr int64_t StreamTimeOutUs{ 2 * 1000 * 1000 };
+		// A send time group holds every packet sent within this time of the first
+		// packet of the group.
+		static constexpr int64_t SendTimeGroupLengthUs{ 5 * 1000 };
+
+		/* Instance methods. */
+
+		DelayBasedBwe::DelayBasedBwe() : DelayBasedBwe(DelayBasedBweOptions{})
+		{
+			MS_TRACE();
+		}
+
+		DelayBasedBwe::DelayBasedBwe(DelayBasedBweOptions options)
+		  : options(options),
+		    videoDelayDetector(std::in_place),
+		    audioDelayDetector(std::in_place),
+		    activeDelayDetector(std::addressof(this->videoDelayDetector.value())),
+		    rateControl(AimdRateControl::AimdRateControlOptions{ .sendSide = true })
+		{
+			MS_TRACE();
+		}
+
+		DelayBasedBwe::Result DelayBasedBwe::IncomingPacketFeedbackVector(
+		  const Types::TransportPacketsFeedback& feedback,
+		  std::optional<int64_t> ackedBitrate,
+		  std::optional<int64_t> probeBitrate,
+		  const std::optional<Types::NetworkStateEstimate>& networkEstimate,
+		  bool inAlr)
+		{
+			MS_TRACE();
+
+			const auto packetResults = feedback.SortedByReceiveTime();
+
+			// NOTE: An empty vector here likely means that every ack arrived too late
+			// and that the send time history had already timed out.
+			if (packetResults.empty())
+			{
+				MS_WARN_TAG(bwe, "very late feedback received");
+
+				return {};
+			}
+
+			bool recoveredFromOveruse{ false };
+			auto prevDetectorState = this->activeDelayDetector->GetState();
+
+			for (const auto& packetResult : packetResults)
+			{
+				IncomingPacketFeedback(packetResult, feedback.feedbackTimeUs);
+
+				if (
+				  prevDetectorState == Types::BandwidthUsage::UNDERUSING &&
+				  this->activeDelayDetector->GetState() == Types::BandwidthUsage::NORMAL)
+				{
+					recoveredFromOveruse = true;
+				}
+
+				prevDetectorState = this->activeDelayDetector->GetState();
+			}
+
+			this->rateControl.SetInApplicationLimitedRegion(inAlr);
+			this->rateControl.SetNetworkStateEstimate(networkEstimate);
+
+			return MaybeUpdateEstimate(
+			  ackedBitrate, probeBitrate, recoveredFromOveruse, feedback.feedbackTimeUs);
+		}
+
+		void DelayBasedBwe::IncomingPacketFeedback(const Types::PacketResult& packetResult, int64_t atTimeUs)
+		{
+			MS_TRACE();
+
+			// NOTE: Safe since only the packets that were received reach this method,
+			// as the caller iterates over `SortedByReceiveTime()`.
+			MS_ASSERT(packetResult.IsReceived(), "packet was not received");
+
+			// NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+			const int64_t receiveTimeUs = packetResult.receiveTimeUs.value();
+
+			// Reset if the stream has timed out, since nothing measured before such a
+			// silence still describes this link.
+			if (!this->lastSeenPacketUs.has_value() || atTimeUs - this->lastSeenPacketUs.value() > StreamTimeOutUs)
+			{
+				this->videoInterArrivalDelta.emplace(SendTimeGroupLengthUs);
+				this->audioInterArrivalDelta.emplace(SendTimeGroupLengthUs);
+				this->videoDelayDetector.emplace();
+				this->audioDelayDetector.emplace();
+
+				this->activeDelayDetector = std::addressof(this->videoDelayDetector.value());
+			}
+
+			this->lastSeenPacketUs = atTimeUs;
+
+			// As an alternative to ignoring small packets, audio and video can be given
+			// a delay detector of their own.
+			// NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+			auto* delayDetectorForPacket = std::addressof(this->videoDelayDetector.value());
+
+			if (this->options.separateAudioPackets)
+			{
+				if (packetResult.sentPacket.audio)
+				{
+					// NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+					delayDetectorForPacket = std::addressof(this->audioDelayDetector.value());
+
+					this->audioPacketsSinceLastVideo++;
+
+					// Video has been silent for long enough that the audio detector is a
+					// better description of the network than the stale video one.
+					if (
+					  this->audioPacketsSinceLastVideo > this->options.separateAudioPacketThreshold &&
+					  (!this->lastVideoPacketRecvTimeUs.has_value() ||
+						 receiveTimeUs - this->lastVideoPacketRecvTimeUs.value() >
+						   this->options.separateAudioTimeThresholdUs))
+					{
+						// NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+						this->activeDelayDetector = std::addressof(this->audioDelayDetector.value());
+					}
+				}
+				else
+				{
+					this->audioPacketsSinceLastVideo = 0;
+
+					this->lastVideoPacketRecvTimeUs =
+					  std::max(this->lastVideoPacketRecvTimeUs.value_or(receiveTimeUs), receiveTimeUs);
+					// NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+					this->activeDelayDetector = std::addressof(this->videoDelayDetector.value());
+				}
+			}
+
+			auto& interArrivalForPacket =
+			  (this->options.separateAudioPackets && packetResult.sentPacket.audio)
+			    // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+			    ? this->audioInterArrivalDelta.value()
+			    // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+			    : this->videoInterArrivalDelta.value();
+
+			const auto deltas = interArrivalForPacket.ComputeDeltas(
+			  packetResult.sentPacket.sendTimeUs, receiveTimeUs, atTimeUs, packetResult.sentPacket.size);
+
+			if (deltas.has_value())
+			{
+				delayDetectorForPacket->Update(
+				  // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+				  deltas.value().sendDeltaUs,
+				  // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+				  deltas.value().arrivalDeltaUs,
+				  receiveTimeUs);
+			}
+		}
+
+		DelayBasedBwe::Result DelayBasedBwe::MaybeUpdateEstimate(
+		  std::optional<int64_t> ackedBitrate,
+		  std::optional<int64_t> probeBitrate,
+		  bool recoveredFromOveruse,
+		  int64_t atTimeUs)
+		{
+			MS_TRACE();
+
+			Result result;
+
+			if (this->activeDelayDetector->GetState() == Types::BandwidthUsage::OVERUSING)
+			{
+				if (
+				  ackedBitrate.has_value() &&
+				  this->rateControl.IsTimeToReduceFurther(atTimeUs, ackedBitrate.value()))
+				{
+					result.updated = UpdateEstimate(atTimeUs, ackedBitrate, result.targetBitrate);
+				}
+				else if (
+				  !ackedBitrate.has_value() && this->rateControl.IsValidEstimate() &&
+				  this->rateControl.IsInitialTimeToReduceFurther(atTimeUs))
+				{
+					// Overusing before any throughput has been measured, so halve the
+					// bitrate instead of aiming at a measurement we don't have.
+					this->rateControl.SetEstimate(this->rateControl.GetLatestEstimate() / 2, atTimeUs);
+
+					result.updated       = true;
+					result.probe         = false;
+					result.targetBitrate = this->rateControl.GetLatestEstimate();
+				}
+			}
+			else
+			{
+				if (probeBitrate.has_value())
+				{
+					result.probe   = true;
+					result.updated = true;
+
+					this->rateControl.SetEstimate(probeBitrate.value(), atTimeUs);
+
+					result.targetBitrate = this->rateControl.GetLatestEstimate();
+				}
+				else
+				{
+					result.updated = UpdateEstimate(atTimeUs, ackedBitrate, result.targetBitrate);
+					result.recoveredFromOveruse = recoveredFromOveruse;
+				}
+			}
+
+			const auto detectorState = this->activeDelayDetector->GetState();
+
+			if ((result.updated && this->prevBitrate != result.targetBitrate) || detectorState != this->prevState)
+			{
+				this->prevBitrate = result.updated ? result.targetBitrate : this->prevBitrate;
+				this->prevState   = detectorState;
+			}
+
+			result.delayDetectorState = detectorState;
+
+			return result;
+		}
+
+		bool DelayBasedBwe::UpdateEstimate(
+		  int64_t atTimeUs, std::optional<int64_t> ackedBitrate, int64_t& targetBitrate)
+		{
+			MS_TRACE();
+
+			const Types::RateControlInput input{ .bandwidthUsage = this->activeDelayDetector->GetState(),
+			                                     .estimatedThroughput = ackedBitrate };
+
+			targetBitrate = this->rateControl.Update(input, atTimeUs);
+
+			return this->rateControl.IsValidEstimate();
+		}
+
+		std::optional<int64_t> DelayBasedBwe::GetLatestEstimate() const
+		{
+			MS_TRACE();
+
+			if (!this->rateControl.IsValidEstimate())
+			{
+				return std::nullopt;
+			}
+
+			return this->rateControl.GetLatestEstimate();
+		}
+
+		void DelayBasedBwe::OnRttUpdate(int64_t avgRttUs)
+		{
+			MS_TRACE();
+
+			this->rateControl.SetRttUs(avgRttUs);
+		}
+
+		void DelayBasedBwe::SetStartBitrate(int64_t startBitrate)
+		{
+			MS_TRACE();
+
+			MS_DEBUG_TAG(bwe, "setting start bitrate [bitrate:%" PRIi64 "]", startBitrate);
+
+			this->rateControl.SetStartBitrate(startBitrate);
+		}
+
+		void DelayBasedBwe::SetMinBitrate(int64_t minBitrate)
+		{
+			MS_TRACE();
+
+			this->rateControl.SetMinBitrate(minBitrate);
+		}
+
+	} // namespace BWE
+} // namespace RTC

package/worker/src/RTC/BWE/FeedbackAdapter.cpp

@@ -0,0 +1,148 @@
+#define MS_CLASS "RTC::BWE::FeedbackAdapter"
+// #define MS_LOG_DEV_LEVEL 3
+
+#include "RTC/BWE/FeedbackAdapter.hpp"
+#include "Logger.hpp"
+
+namespace RTC
+{
+	namespace BWE
+	{
+		/* Instance methods. */
+
+		FeedbackAdapter::FeedbackAdapter(SendPacketHistory* sendPacketHistory)
+		  : sendPacketHistory(sendPacketHistory)
+		{
+			MS_TRACE();
+		}
+
+		std::optional<Types::TransportPacketsFeedback> FeedbackAdapter::ProcessTransportFeedback(
+		  const RTCP::FeedbackRtpTransportPacket* feedback, int64_t receivedAtUs)
+		{
+			MS_TRACE();
+
+			if (feedback->GetPacketStatusCount() == 0)
+			{
+				MS_DEBUG_TAG(bwe, "feedback reports on no packet at all");
+
+				return std::nullopt;
+			}
+
+			const auto lastSequenceNumber = this->sendPacketHistory->GetLastSequenceNumber();
+
+			if (!lastSequenceNumber.has_value())
+			{
+				MS_DEBUG_TAG(bwe, "feedback received before any packet was sent");
+
+				return std::nullopt;
+			}
+
+			const int64_t lastSentSequenceNumber = lastSequenceNumber.value();
+
+			const int64_t baseTimeUs = feedback->GetReferenceTimestampUs();
+
+			if (!this->lastFeedbackBaseTimeUs.has_value())
+			{
+				this->currentOffsetUs = receivedAtUs;
+			}
+			else
+			{
+				const int64_t baseDeltaUs = feedback->GetBaseDeltaUs(this->lastFeedbackBaseTimeUs.value());
+
+				if (baseDeltaUs < -this->currentOffsetUs)
+				{
+					MS_DEBUG_TAG(
+					  bwe,
+					  "base time of the feedback moved too far back, anchoring it again"
+					  " [baseDelta:%" PRIi64 " us]",
+					  baseDeltaUs);
+
+					this->currentOffsetUs = receivedAtUs;
+				}
+				else
+				{
+					this->currentOffsetUs += baseDeltaUs;
+				}
+			}
+
+			this->lastFeedbackBaseTimeUs = baseTimeUs;
+
+			const auto packetStatuses = feedback->GetPacketStatuses();
+
+			Types::TransportPacketsFeedback transportPacketsFeedback;
+
+			transportPacketsFeedback.feedbackTimeUs = receivedAtUs;
+			transportPacketsFeedback.packetFeedbacks.reserve(packetStatuses.size());
+
+			size_t failedLookups{ 0 };
+
+			// Each packet is resolved against the one reported before it, since the
+			// packets being sent may be far ahead of the ones the feedbacks report on.
+			// The latest packet sent seeds it, and bounds it as well, since no report
+			// can stand for a packet that was never sent.
+			int64_t sequenceNumber = std::min(
+			  this->lastUnwrappedSequenceNumber.value_or(lastSentSequenceNumber), lastSentSequenceNumber);
+
+			for (const auto& packetStatus : packetStatuses)
+			{
+				sequenceNumber = UnwrapSequenceNumber(packetStatus.sequenceNumber, sequenceNumber);
+
+				const auto sentPacket =
+				  this->sendPacketHistory->RetrievePacket(sequenceNumber, packetStatus.received);
+
+				if (!sentPacket.has_value())
+				{
+					++failedLookups;
+
+					continue;
+				}
+
+				Types::PacketResult packetResult;
+
+				packetResult.sentPacket = sentPacket.value();
+
+				if (packetStatus.received)
+				{
+					packetResult.receiveTimeUs =
+					  this->currentOffsetUs + (packetStatus.receivedAtUs - baseTimeUs);
+				}
+
+				transportPacketsFeedback.packetFeedbacks.push_back(packetResult);
+			}
+
+			this->lastUnwrappedSequenceNumber = sequenceNumber;
+
+			if (failedLookups > 0)
+			{
+				MS_DEBUG_TAG(
+				  bwe,
+				  "could not resolve %zu of the %zu reported packets, packets reordered or history too small",
+				  failedLookups,
+				  packetStatuses.size());
+			}
+
+			if (transportPacketsFeedback.packetFeedbacks.empty())
+			{
+				return std::nullopt;
+			}
+
+			transportPacketsFeedback.dataInFlight = this->sendPacketHistory->GetOutstandingBytes();
+
+			return transportPacketsFeedback;
+		}
+
+		int64_t FeedbackAdapter::UnwrapSequenceNumber(
+		  uint16_t wideSequenceNumber, int64_t previousSequenceNumber)
+		{
+			MS_TRACE();
+
+			// The wire carries the sequence number of the history truncated to 16 bits,
+			// so the closest value to the previous one ending in those very same bits
+			// is the one it stands for.
+			const auto deltaSequenceNumber = static_cast<int16_t>(
+			  static_cast<uint16_t>(wideSequenceNumber - static_cast<uint16_t>(previousSequenceNumber)));
+
+			return previousSequenceNumber + deltaSequenceNumber;
+		}
+	} // namespace BWE
+} // namespace RTC

package/worker/src/RTC/BWE/InterArrivalDelta.cpp

@@ -0,0 +1,181 @@
+#define MS_CLASS "RTC::BWE::InterArrivalDelta"
+// #define MS_LOG_DEV_LEVEL 3
+
+#include "RTC/BWE/InterArrivalDelta.hpp"
+#include "Logger.hpp"
+
+namespace RTC
+{
+	namespace BWE
+	{
+		/* Static. */
+
+		// Maximum arrival time delta for a packet to be absorbed into the ongoing
+		// burst.
+		static constexpr int64_t BurstDeltaThresholdUs{ 5000 };
+		// Maximum duration of a burst.
+		static constexpr int64_t MaxBurstDurationUs{ 100000 };
+		// Number of consecutive reordered groups after which the state is reset.
+		static constexpr size_t ReorderedResetThreshold{ 3 };
+		// Difference between the arrival time delta and our own time delta after
+		// which the state is reset, assuming that the remote clock has jumped.
+		static constexpr int64_t ArrivalTimeOffsetThresholdUs{ 3000000 };
+
+		/* Instance methods. */
+
+		InterArrivalDelta::InterArrivalDelta(int64_t sendTimeGroupLengthUs)
+		  : sendTimeGroupLengthUs(sendTimeGroupLengthUs)
+		{
+			MS_TRACE();
+		}
+
+		std::optional<InterArrivalDelta::Deltas> InterArrivalDelta::ComputeDeltas(
+		  int64_t sendTimeUs, int64_t arrivalTimeUs, int64_t feedbackAtUs, size_t packetSize)
+		{
+			MS_TRACE();
+
+			std::optional<Deltas> deltas;
+
+			if (this->currentGroup.IsFirstPacket())
+			{
+				// Not enough data to compute deltas yet, so just store the packet until
+				// there are two groups to compare.
+				this->currentGroup.firstSendTimeUs    = sendTimeUs;
+				this->currentGroup.sendTimeUs         = sendTimeUs;
+				this->currentGroup.firstArrivalTimeUs = arrivalTimeUs;
+			}
+			else if (this->currentGroup.firstSendTimeUs > sendTimeUs)
+			{
+				// Reordered packet.
+				return std::nullopt;
+			}
+			else if (IsNewSendTimeGroup(arrivalTimeUs, sendTimeUs))
+			{
+				// This is the first packet of a later send burst, so the sample of the
+				// ongoing group is ready.
+				if (!this->prevGroup.IsFirstPacket())
+				{
+					const int64_t sendDeltaUs = this->currentGroup.sendTimeUs - this->prevGroup.sendTimeUs;
+					const int64_t arrivalDeltaUs =
+					  // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+					  this->currentGroup.completeTimeUs.value() -
+					  // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+					  this->prevGroup.completeTimeUs.value();
+					const int64_t ownDeltaUs =
+					  this->currentGroup.lastFeedbackAtUs - this->prevGroup.lastFeedbackAtUs;
+
+					if (arrivalDeltaUs - ownDeltaUs >= ArrivalTimeOffsetThresholdUs)
+					{
+						MS_WARN_TAG(
+						  bwe,
+						  "arrival time clock offset has changed, resetting [diff:%" PRIi64 " us]",
+						  arrivalDeltaUs - ownDeltaUs);
+
+						Reset();
+
+						return std::nullopt;
+					}
+
+					if (arrivalDeltaUs < 0)
+					{
+						// The groups have been reordered after their arrival time was taken.
+						++this->numConsecutiveReorderedGroups;
+
+						if (this->numConsecutiveReorderedGroups >= ReorderedResetThreshold)
+						{
+							MS_WARN_TAG(
+							  bwe,
+							  "packets between send bursts arrived out of order, resetting [arrivalDelta:%" PRIi64
+							  " us, sendDelta:%" PRIi64 " us]",
+							  arrivalDeltaUs,
+							  sendDeltaUs);
+
+							Reset();
+						}
+
+						return std::nullopt;
+					}
+
+					this->numConsecutiveReorderedGroups = 0;
+
+					deltas = Deltas{ .sendDeltaUs    = sendDeltaUs,
+					                 .arrivalDeltaUs = arrivalDeltaUs,
+					                 .sizeDelta      = static_cast<int64_t>(this->currentGroup.size) -
+						                                 static_cast<int64_t>(this->prevGroup.size) };
+				}
+
+				this->prevGroup = this->currentGroup;
+
+				// The new packet starts the current group.
+				this->currentGroup.firstSendTimeUs    = sendTimeUs;
+				this->currentGroup.sendTimeUs         = sendTimeUs;
+				this->currentGroup.firstArrivalTimeUs = arrivalTimeUs;
+				this->currentGroup.size               = 0;
+			}
+			else
+			{
+				this->currentGroup.sendTimeUs = std::max(this->currentGroup.sendTimeUs, sendTimeUs);
+			}
+
+			// Accumulate the group size.
+			this->currentGroup.size += packetSize;
+			this->currentGroup.completeTimeUs   = arrivalTimeUs;
+			this->currentGroup.lastFeedbackAtUs = feedbackAtUs;
+
+			return deltas;
+		}
+
+		bool InterArrivalDelta::IsNewSendTimeGroup(int64_t arrivalTimeUs, int64_t sendTimeUs) const
+		{
+			MS_TRACE();
+
+			if (this->currentGroup.IsFirstPacket())
+			{
+				return false;
+			}
+			else if (BelongsToBurst(arrivalTimeUs, sendTimeUs))
+			{
+				return false;
+			}
+			else
+			{
+				return sendTimeUs - this->currentGroup.firstSendTimeUs > this->sendTimeGroupLengthUs;
+			}
+		}
+
+		bool InterArrivalDelta::BelongsToBurst(int64_t arrivalTimeUs, int64_t sendTimeUs) const
+		{
+			MS_TRACE();
+
+			MS_ASSERT(!this->currentGroup.IsFirstPacket(), "current group is empty");
+
+			const int64_t arrivalDeltaUs =
+			  // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
+			  arrivalTimeUs - this->currentGroup.completeTimeUs.value();
+			const int64_t sendDeltaUs = sendTimeUs - this->currentGroup.sendTimeUs;
+
+			// Packets sent at the very same time always belong to the same burst.
+			if (sendDeltaUs == 0)
+			{
+				return true;
+			}
+
+			const int64_t propagationDeltaUs = arrivalDeltaUs - sendDeltaUs;
+			const int64_t burstDurationUs    = arrivalTimeUs - this->currentGroup.firstArrivalTimeUs;
+
+			// The packet arrived earlier than its send time gap would suggest, meaning
+			// that it was serialized by the network together with the ongoing group.
+			return propagationDeltaUs < 0 && arrivalDeltaUs <= BurstDeltaThresholdUs &&
+			       burstDurationUs < MaxBurstDurationUs;
+		}
+
+		void InterArrivalDelta::Reset()
+		{
+			MS_TRACE();
+
+			this->numConsecutiveReorderedGroups = 0;
+			this->currentGroup                  = SendTimeGroup();
+			this->prevGroup                     = SendTimeGroup();
+		}
+	} // namespace BWE
+} // namespace RTC