mediasoup 3.20.4 → 3.20.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/package.json +4 -4
  2. package/worker/include/RTC/Consumer.hpp +80 -45
  3. package/worker/include/RTC/PipeProducerStreamManager.hpp +78 -0
  4. package/worker/include/RTC/ProducerStreamManager.hpp +181 -0
  5. package/worker/include/RTC/SCTP/association/Association.hpp +24 -0
  6. package/worker/include/RTC/SCTP/association/StateCookie.hpp +104 -11
  7. package/worker/include/RTC/SCTP/packet/parameters/StateCookieParameter.hpp +4 -1
  8. package/worker/include/RTC/SCTP/public/SctpOptions.hpp +13 -0
  9. package/worker/include/RTC/SeqManager.hpp +2 -2
  10. package/worker/include/RTC/SimpleProducerStreamManager.hpp +72 -0
  11. package/worker/include/RTC/SimulcastProducerStreamManager.hpp +93 -0
  12. package/worker/include/RTC/SvcProducerStreamManager.hpp +72 -0
  13. package/worker/include/RTC/Transport.hpp +7 -1
  14. package/worker/meson.build +9 -5
  15. package/worker/src/RTC/Consumer.cpp +1404 -30
  16. package/worker/src/RTC/DirectTransport.cpp +4 -1
  17. package/worker/src/RTC/PipeProducerStreamManager.cpp +266 -0
  18. package/worker/src/RTC/PipeTransport.cpp +4 -1
  19. package/worker/src/RTC/PlainTransport.cpp +4 -1
  20. package/worker/src/RTC/SCTP/association/Association.cpp +142 -3
  21. package/worker/src/RTC/SCTP/association/StateCookie.cpp +96 -31
  22. package/worker/src/RTC/SCTP/association/StreamResetHandler.cpp +4 -0
  23. package/worker/src/RTC/SCTP/packet/Packet.cpp +1 -1
  24. package/worker/src/RTC/SCTP/packet/parameters/StateCookieParameter.cpp +12 -3
  25. package/worker/src/RTC/SCTP/public/SctpOptions.cpp +4 -0
  26. package/worker/src/RTC/SCTP/rx/DataTracker.cpp +4 -1
  27. package/worker/src/RTC/SeqManager.cpp +42 -29
  28. package/worker/src/RTC/SimpleProducerStreamManager.cpp +343 -0
  29. package/worker/src/RTC/SimulcastProducerStreamManager.cpp +1068 -0
  30. package/worker/src/RTC/SvcProducerStreamManager.cpp +664 -0
  31. package/worker/src/RTC/Transport.cpp +7 -44
  32. package/worker/src/RTC/WebRtcTransport.cpp +8 -2
  33. package/worker/test/include/RTC/SCTP/sctpCommon.hpp +1 -1
  34. package/worker/test/src/RTC/SCTP/association/TestAssociation.cpp +115 -0
  35. package/worker/test/src/RTC/SCTP/association/TestStateCookie.cpp +123 -0
  36. package/worker/test/src/RTC/SCTP/packet/TestPacket.cpp +4 -4
  37. package/worker/test/src/RTC/{TestSimpleConsumer.cpp → TestConsumer.cpp} +6 -7
  38. package/worker/test/src/RTC/TestPipeProducerStreamManager.cpp +471 -0
  39. package/worker/test/src/RTC/TestSimpleProducerStreamManager.cpp +531 -0
  40. package/worker/test/src/RTC/TestSimulcastProducerStreamManager.cpp +1040 -0
  41. package/worker/test/src/RTC/TestSvcProducerStreamManager.cpp +1278 -0
  42. package/worker/include/RTC/PipeConsumer.hpp +0 -95
  43. package/worker/include/RTC/SimpleConsumer.hpp +0 -102
  44. package/worker/include/RTC/SimulcastConsumer.hpp +0 -141
  45. package/worker/include/RTC/SvcConsumer.hpp +0 -118
  46. package/worker/src/RTC/PipeConsumer.cpp +0 -874
  47. package/worker/src/RTC/SimpleConsumer.cpp +0 -882
  48. package/worker/src/RTC/SimulcastConsumer.cpp +0 -1887
  49. package/worker/src/RTC/SvcConsumer.cpp +0 -1384
@@ -4,27 +4,42 @@
4
4
  #include "RTC/Consumer.hpp"
5
5
  #include "Logger.hpp"
6
6
  #include "MediaSoupErrors.hpp"
7
+ #include "RTC/PipeProducerStreamManager.hpp"
8
+ #include "RTC/RTP/Codecs/Tools.hpp"
9
+ #include "RTC/SimpleProducerStreamManager.hpp"
10
+ #include "RTC/SimulcastProducerStreamManager.hpp"
11
+ #include "RTC/SvcProducerStreamManager.hpp"
12
+ #include "Utils.hpp"
13
+ #ifdef MS_RTC_LOGGER_RTP
14
+ #include "RTC/RtcLogger.hpp"
15
+ #endif
16
+ #include <limits> // std::numeric_limits
7
17
 
8
18
  namespace RTC
9
19
  {
20
+ /* Static. */
21
+
22
+ static constexpr size_t TargetLayerRetransmissionBufferSize{ 30u };
23
+
10
24
  /* Instance methods. */
11
25
 
12
26
  Consumer::Consumer(
13
27
  SharedInterface* shared,
14
28
  const std::string& id,
15
29
  const std::string& producerId,
16
- Listener* listener,
17
- const FBS::Transport::ConsumeRequest* data,
18
- RTC::RtpParameters::Type type)
30
+ RTC::Consumer::Listener* listener,
31
+ const FBS::Transport::ConsumeRequest* data)
19
32
  : id(id),
20
33
  producerId(producerId),
21
34
  shared(shared),
22
35
  listener(listener),
23
36
  kind(RTC::Media::Kind(data->kind())),
24
- type(type)
37
+ type(RTC::RtpParameters::Type(data->type()))
25
38
  {
26
39
  MS_TRACE();
27
40
 
41
+ this->pipe = this->type == RTC::RtpParameters::Type::PIPE;
42
+
28
43
  // This may throw.
29
44
  this->rtpParameters = RTC::RtpParameters(data->rtpParameters());
30
45
 
@@ -69,6 +84,12 @@ namespace RTC
69
84
  }
70
85
  }
71
86
 
87
+ // Ensure there are as many encodings as consumable encodings for pipe.
88
+ if (pipe && this->rtpParameters.encodings.size() != this->consumableRtpEncodings.size())
89
+ {
90
+ MS_THROW_TYPE_ERROR("number of rtpParameters.encodings and consumableRtpEncodings do not match");
91
+ }
92
+
72
93
  // Fill RTP header extension ids and their mapped values.
73
94
  // This may throw.
74
95
  for (auto& exten : this->rtpParameters.headerExtensions)
@@ -120,11 +141,6 @@ namespace RTC
120
141
  this->rtpHeaderExtensionIds.videoOrientation = exten.id;
121
142
  }
122
143
 
123
- if (this->rtpHeaderExtensionIds.videoOrientation == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::VIDEO_ORIENTATION)
124
- {
125
- this->rtpHeaderExtensionIds.videoOrientation = exten.id;
126
- }
127
-
128
144
  if (this->rtpHeaderExtensionIds.absCaptureTime == 0u && exten.type == RTC::RtpHeaderExtensionUri::Type::ABS_CAPTURE_TIME)
129
145
  {
130
146
  this->rtpHeaderExtensionIds.absCaptureTime = exten.id;
@@ -177,14 +193,294 @@ namespace RTC
177
193
  {
178
194
  this->maxRtcpInterval = RTC::RTCP::MaxVideoIntervalMs;
179
195
  }
196
+
197
+ auto& encoding = this->rtpParameters.encodings[0];
198
+
199
+ // Determine keyFrameSupported.
200
+ const auto* mediaCodec = this->rtpParameters.GetCodecForEncoding(encoding);
201
+ const bool keyFrameSupported = RTC::RTP::Codecs::Tools::CanBeKeyFrame(mediaCodec->mimeType);
202
+
203
+ // Build preferred layers from FBS data.
204
+ RTC::ConsumerTypes::VideoLayers preferredLayers;
205
+
206
+ // Create the appropriate ProducerStreamManager subclass based on type.
207
+ switch (this->type)
208
+ {
209
+ case RTC::RtpParameters::Type::SIMPLE:
210
+ {
211
+ // Ensure there is a single encoding.
212
+ if (this->consumableRtpEncodings.size() != 1u)
213
+ {
214
+ MS_THROW_TYPE_ERROR("invalid consumableRtpEncodings with size != 1");
215
+ }
216
+
217
+ // Create the encoding context for Opus (DTX filtering).
218
+ std::unique_ptr<RTC::RTP::Codecs::EncodingContext> encodingContext;
219
+
220
+ if (
221
+ mediaCodec->mimeType.type == RTC::RtpCodecMimeType::Type::AUDIO &&
222
+ (mediaCodec->mimeType.subtype == RTC::RtpCodecMimeType::Subtype::OPUS ||
223
+ mediaCodec->mimeType.subtype == RTC::RtpCodecMimeType::Subtype::MULTIOPUS))
224
+ {
225
+ RTC::RTP::Codecs::EncodingContext::Params params;
226
+
227
+ encodingContext.reset(
228
+ RTC::RTP::Codecs::Tools::GetEncodingContext(mediaCodec->mimeType, params));
229
+
230
+ // ignoreDtx is set to false by default.
231
+ encodingContext->SetIgnoreDtx(data->ignoreDtx());
232
+ }
233
+
234
+ this->producerStreamManager = std::make_unique<SimpleProducerStreamManager>(
235
+ this->consumableRtpEncodings,
236
+ preferredLayers,
237
+ std::move(encodingContext),
238
+ this->kind,
239
+ keyFrameSupported,
240
+ this,
241
+ this->shared);
242
+
243
+ break;
244
+ }
245
+
246
+ case RTC::RtpParameters::Type::SIMULCAST:
247
+ {
248
+ // We allow a single encoding in simulcast (so we can enable temporal
249
+ // layers with a single simulcast stream).
250
+
251
+ // Ensure there are as many spatial layers as encodings.
252
+ if (encoding.spatialLayers != this->consumableRtpEncodings.size())
253
+ {
254
+ MS_THROW_TYPE_ERROR(
255
+ "encoding.spatialLayers does not match number of consumableRtpEncodings");
256
+ }
257
+
258
+ // Set preferredLayers.
259
+ if (flatbuffers::IsFieldPresent(data, FBS::Transport::ConsumeRequest::VT_PREFERREDLAYERS))
260
+ {
261
+ const auto* fbsPreferredLayers = data->preferredLayers();
262
+
263
+ preferredLayers.spatial = fbsPreferredLayers->spatialLayer();
264
+
265
+ if (preferredLayers.spatial > encoding.spatialLayers - 1)
266
+ {
267
+ preferredLayers.spatial = static_cast<int16_t>(encoding.spatialLayers - 1);
268
+ }
269
+
270
+ if (
271
+ auto preferredTemporalLayer = fbsPreferredLayers->temporalLayer();
272
+ preferredTemporalLayer.has_value())
273
+ {
274
+ preferredLayers.temporal = preferredTemporalLayer.value();
275
+
276
+ if (preferredLayers.temporal > encoding.temporalLayers - 1)
277
+ {
278
+ preferredLayers.temporal = static_cast<int16_t>(encoding.temporalLayers - 1);
279
+ }
280
+ }
281
+ else
282
+ {
283
+ preferredLayers.temporal = static_cast<int16_t>(encoding.temporalLayers - 1);
284
+ }
285
+ }
286
+ else
287
+ {
288
+ // Initially set preferredSpatialLayer and preferredTemporalLayer
289
+ // to the maximum value.
290
+ preferredLayers.spatial = static_cast<int16_t>(encoding.spatialLayers - 1);
291
+ preferredLayers.temporal = static_cast<int16_t>(encoding.temporalLayers - 1);
292
+ }
293
+
294
+ // Create the encoding context.
295
+ if (!RTC::RTP::Codecs::Tools::IsValidTypeForCodec(this->type, mediaCodec->mimeType))
296
+ {
297
+ MS_THROW_TYPE_ERROR(
298
+ "%s codec not supported for simulcast", mediaCodec->mimeType.ToString().c_str());
299
+ }
300
+
301
+ RTC::RTP::Codecs::EncodingContext::Params params;
302
+
303
+ params.spatialLayers = encoding.spatialLayers;
304
+ params.temporalLayers = encoding.temporalLayers;
305
+
306
+ std::unique_ptr<RTC::RTP::Codecs::EncodingContext> encodingContext(
307
+ RTC::RTP::Codecs::Tools::GetEncodingContext(mediaCodec->mimeType, params));
308
+
309
+ MS_ASSERT(encodingContext, "no encoding context for this codec");
310
+
311
+ this->producerStreamManager = std::make_unique<SimulcastProducerStreamManager>(
312
+ this->consumableRtpEncodings,
313
+ preferredLayers,
314
+ std::move(encodingContext),
315
+ this->kind,
316
+ keyFrameSupported,
317
+ this,
318
+ this->shared);
319
+
320
+ break;
321
+ }
322
+
323
+ case RTC::RtpParameters::Type::SVC:
324
+ {
325
+ // Ensure there is a single encoding.
326
+ if (this->consumableRtpEncodings.size() != 1u)
327
+ {
328
+ MS_THROW_TYPE_ERROR("invalid consumableRtpEncodings with size != 1");
329
+ }
330
+
331
+ // Ensure there are multiple spatial or temporal layers.
332
+ if (encoding.spatialLayers < 2u && encoding.temporalLayers < 2u)
333
+ {
334
+ MS_THROW_TYPE_ERROR("invalid number of layers");
335
+ }
336
+
337
+ // Set preferredLayers.
338
+ if (flatbuffers::IsFieldPresent(data, FBS::Transport::ConsumeRequest::VT_PREFERREDLAYERS))
339
+ {
340
+ preferredLayers.spatial = data->preferredLayers()->spatialLayer();
341
+
342
+ if (preferredLayers.spatial > encoding.spatialLayers - 1)
343
+ {
344
+ preferredLayers.spatial = static_cast<int16_t>(encoding.spatialLayers - 1);
345
+ }
346
+
347
+ // preferredTemporalLayer is optional.
348
+ auto preferredTemporalLayer = data->preferredLayers()->temporalLayer();
349
+
350
+ if (preferredTemporalLayer)
351
+ {
352
+ preferredLayers.temporal = preferredTemporalLayer.value();
353
+
354
+ if (preferredLayers.temporal > encoding.temporalLayers - 1)
355
+ {
356
+ preferredLayers.temporal = static_cast<int16_t>(encoding.temporalLayers - 1);
357
+ }
358
+ }
359
+ else
360
+ {
361
+ preferredLayers.temporal = static_cast<int16_t>(encoding.temporalLayers - 1);
362
+ }
363
+ }
364
+ else
365
+ {
366
+ // Initially set preferredSpatialLayer and preferredTemporalLayer
367
+ // to the maximum value.
368
+ preferredLayers.spatial = static_cast<int16_t>(encoding.spatialLayers - 1);
369
+ preferredLayers.temporal = static_cast<int16_t>(encoding.temporalLayers - 1);
370
+ }
371
+
372
+ // Create the encoding context.
373
+ if (!RTC::RTP::Codecs::Tools::IsValidTypeForCodec(this->type, mediaCodec->mimeType))
374
+ {
375
+ MS_THROW_TYPE_ERROR(
376
+ "%s codec not supported for svc", mediaCodec->mimeType.ToString().c_str());
377
+ }
378
+
379
+ RTC::RTP::Codecs::EncodingContext::Params params;
380
+
381
+ params.spatialLayers = encoding.spatialLayers;
382
+ params.temporalLayers = encoding.temporalLayers;
383
+ params.ksvc = encoding.ksvc;
384
+
385
+ std::unique_ptr<RTC::RTP::Codecs::EncodingContext> encodingContext(
386
+ RTC::RTP::Codecs::Tools::GetEncodingContext(mediaCodec->mimeType, params));
387
+
388
+ MS_ASSERT(encodingContext, "no encoding context for this codec");
389
+
390
+ this->producerStreamManager = std::make_unique<SvcProducerStreamManager>(
391
+ this->consumableRtpEncodings,
392
+ preferredLayers,
393
+ std::move(encodingContext),
394
+ this->kind,
395
+ keyFrameSupported,
396
+ this,
397
+ this->shared);
398
+
399
+ break;
400
+ }
401
+
402
+ case RTC::RtpParameters::Type::PIPE:
403
+ {
404
+ // Pipe consumer: no layer management, N streams.
405
+ this->producerStreamManager = std::make_unique<PipeProducerStreamManager>(
406
+ this->consumableRtpEncodings,
407
+ preferredLayers,
408
+ nullptr,
409
+ this->kind,
410
+ keyFrameSupported,
411
+ this,
412
+ this->shared);
413
+
414
+ break;
415
+ }
416
+
417
+ default:
418
+ {
419
+ MS_THROW_TYPE_ERROR("invalid consumer type");
420
+ }
421
+ }
422
+
423
+ // Create RtpStreamSend instances.
424
+ CreateRtpStreams();
425
+
426
+ // NOTE: This may throw.
427
+ this->shared->GetChannelMessageRegistrator()->RegisterHandler(
428
+ this->id,
429
+ /*channelRequestHandler*/ this,
430
+ /*channelNotificationHandler*/ nullptr);
180
431
  }
181
432
 
182
433
  Consumer::~Consumer()
183
434
  {
184
435
  MS_TRACE();
436
+
437
+ this->shared->GetChannelMessageRegistrator()->UnregisterHandler(this->id);
438
+
439
+ for (auto* rtpStream : this->rtpStreams)
440
+ {
441
+ delete rtpStream;
442
+ }
443
+
444
+ this->rtpStreams.clear();
445
+ this->mapMappedSsrcSsrc.clear();
446
+ this->mapSsrcRtpStream.clear();
447
+ this->mapRtpStreamRtpSeqManager.clear();
448
+ this->mapRtpStreamTargetLayerRetransmissionBuffer.clear();
449
+ }
450
+
451
+ flatbuffers::Offset<FBS::Consumer::DumpResponse> Consumer::FillBuffer(
452
+ flatbuffers::FlatBufferBuilder& builder) const
453
+ {
454
+ MS_TRACE();
455
+
456
+ // Call the base method.
457
+ auto base = FillBufferBase(builder);
458
+ // Add rtpStreams.
459
+ std::vector<flatbuffers::Offset<FBS::RtpStream::Dump>> rtpStreams;
460
+ rtpStreams.reserve(this->rtpStreams.size());
461
+
462
+ for (const auto* rtpStream : this->rtpStreams)
463
+ {
464
+ rtpStreams.emplace_back(rtpStream->FillBuffer(builder));
465
+ }
466
+
467
+ auto targetLayers = this->producerStreamManager->GetTargetLayers();
468
+
469
+ auto dump = FBS::Consumer::CreateConsumerDumpDirect(
470
+ builder,
471
+ base,
472
+ &rtpStreams,
473
+ this->producerStreamManager->GetPreferredLayers().spatial,
474
+ targetLayers.spatial,
475
+ this->producerStreamManager->GetCurrentSpatialLayer(),
476
+ this->producerStreamManager->GetPreferredLayers().temporal,
477
+ targetLayers.temporal,
478
+ this->producerStreamManager->GetCurrentTemporalLayer());
479
+
480
+ return FBS::Consumer::CreateDumpResponse(builder, dump);
185
481
  }
186
482
 
187
- flatbuffers::Offset<FBS::Consumer::BaseConsumerDump> Consumer::FillBuffer(
483
+ flatbuffers::Offset<FBS::Consumer::BaseConsumerDump> Consumer::FillBufferBase(
188
484
  flatbuffers::FlatBufferBuilder& builder) const
189
485
  {
190
486
  MS_TRACE();
@@ -252,12 +548,173 @@ namespace RTC
252
548
  this->priority);
253
549
  }
254
550
 
551
+ flatbuffers::Offset<FBS::Consumer::GetStatsResponse> Consumer::FillBufferStats(
552
+ flatbuffers::FlatBufferBuilder& builder)
553
+ {
554
+ MS_TRACE();
555
+
556
+ std::vector<flatbuffers::Offset<FBS::RtpStream::Stats>> rtpStreams;
557
+ rtpStreams.reserve(this->rtpStreams.size());
558
+
559
+ // Add stats of our send streams.
560
+ for (auto* rtpStream : this->rtpStreams)
561
+ {
562
+ rtpStreams.emplace_back(rtpStream->FillBufferStats(builder));
563
+ }
564
+
565
+ // Add stats of the current recv stream.
566
+ auto* producerCurrentRtpStream = this->producerStreamManager->GetProducerCurrentRtpStream();
567
+
568
+ if (producerCurrentRtpStream)
569
+ {
570
+ rtpStreams.emplace_back(producerCurrentRtpStream->FillBufferStats(builder));
571
+ }
572
+
573
+ return FBS::Consumer::CreateGetStatsResponseDirect(builder, &rtpStreams);
574
+ }
575
+
576
+ flatbuffers::Offset<FBS::Consumer::ConsumerScore> Consumer::FillBufferScore(
577
+ flatbuffers::FlatBufferBuilder& builder) const
578
+ {
579
+ MS_TRACE();
580
+
581
+ MS_ASSERT(this->producerRtpStreamScores, "producerRtpStreamScores not set");
582
+
583
+ // NOTE: Hardcoded values in PipeTransport.
584
+ if (this->pipe)
585
+ {
586
+ return FBS::Consumer::CreateConsumerScoreDirect(builder, 10, 10, this->producerRtpStreamScores);
587
+ }
588
+
589
+ auto* rtpStream = this->mapSsrcRtpStream.begin()->second;
590
+ uint8_t producerScore{ 0 };
591
+
592
+ auto* producerCurrentRtpStream = this->producerStreamManager->GetProducerCurrentRtpStream();
593
+
594
+ if (producerCurrentRtpStream)
595
+ {
596
+ producerScore = producerCurrentRtpStream->GetScore();
597
+ }
598
+
599
+ return FBS::Consumer::CreateConsumerScoreDirect(
600
+ builder, rtpStream->GetScore(), producerScore, this->producerRtpStreamScores);
601
+ }
602
+
255
603
  void Consumer::HandleRequest(Channel::ChannelRequest* request)
256
604
  {
257
605
  MS_TRACE();
258
606
 
259
607
  switch (request->method)
260
608
  {
609
+ case Channel::ChannelRequest::Method::CONSUMER_DUMP:
610
+ {
611
+ auto dumpOffset = FillBuffer(request->GetBufferBuilder());
612
+
613
+ request->Accept(FBS::Response::Body::Consumer_DumpResponse, dumpOffset);
614
+
615
+ break;
616
+ }
617
+
618
+ case Channel::ChannelRequest::Method::CONSUMER_REQUEST_KEY_FRAME:
619
+ {
620
+ if (IsActive())
621
+ {
622
+ if (this->pipe)
623
+ {
624
+ if (this->kind != RTC::Media::Kind::VIDEO)
625
+ {
626
+ return;
627
+ }
628
+
629
+ for (auto& consumableRtpEncoding : this->consumableRtpEncodings)
630
+ {
631
+ auto mappedSsrc = consumableRtpEncoding.ssrc;
632
+
633
+ this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc);
634
+ }
635
+ }
636
+ else
637
+ {
638
+ this->producerStreamManager->RequestKeyFrame();
639
+ }
640
+ }
641
+
642
+ request->Accept();
643
+
644
+ break;
645
+ }
646
+
647
+ case Channel::ChannelRequest::Method::CONSUMER_SET_PREFERRED_LAYERS:
648
+ {
649
+ // Simple consumers and pipes have no layers concept.
650
+ if (this->type == RTC::RtpParameters::Type::SIMPLE || this->pipe)
651
+ {
652
+ // Accept with empty preferred layers object.
653
+ auto responseOffset =
654
+ FBS::Consumer::CreateSetPreferredLayersResponse(request->GetBufferBuilder());
655
+
656
+ request->Accept(FBS::Response::Body::Consumer_SetPreferredLayersResponse, responseOffset);
657
+
658
+ break;
659
+ }
660
+
661
+ auto* rtpStream = this->mapSsrcRtpStream.begin()->second;
662
+ auto previousPreferredLayers = this->producerStreamManager->GetPreferredLayers();
663
+
664
+ const auto* body = request->data->body_as<FBS::Consumer::SetPreferredLayersRequest>();
665
+ const auto* preferredLayers = body->preferredLayers();
666
+
667
+ RTC::ConsumerTypes::VideoLayers newPreferredLayers;
668
+
669
+ // Spatial layer.
670
+ newPreferredLayers.spatial = preferredLayers->spatialLayer();
671
+
672
+ if (newPreferredLayers.spatial > rtpStream->GetSpatialLayers() - 1)
673
+ {
674
+ newPreferredLayers.spatial = static_cast<int16_t>(rtpStream->GetSpatialLayers() - 1);
675
+ }
676
+
677
+ // preferredTemporalLayer is optional.
678
+ auto preferredTemporalLayer = preferredLayers->temporalLayer();
679
+
680
+ if (preferredTemporalLayer.has_value())
681
+ {
682
+ newPreferredLayers.temporal = preferredTemporalLayer.value();
683
+
684
+ if (newPreferredLayers.temporal > rtpStream->GetTemporalLayers() - 1)
685
+ {
686
+ newPreferredLayers.temporal = static_cast<int16_t>(rtpStream->GetTemporalLayers() - 1);
687
+ }
688
+ }
689
+ else
690
+ {
691
+ newPreferredLayers.temporal = static_cast<int16_t>(rtpStream->GetTemporalLayers() - 1);
692
+ }
693
+
694
+ this->producerStreamManager->SetPreferredLayers(newPreferredLayers);
695
+
696
+ MS_DEBUG_DEV(
697
+ "preferred layers changed [spatial:%" PRIi16 ", temporal:%" PRIi16 ", consumerId:%s]",
698
+ newPreferredLayers.spatial,
699
+ newPreferredLayers.temporal,
700
+ this->id.c_str());
701
+
702
+ preferredTemporalLayer = newPreferredLayers.temporal;
703
+ auto preferredLayersOffset = FBS::Consumer::CreateConsumerLayers(
704
+ request->GetBufferBuilder(), newPreferredLayers.spatial, preferredTemporalLayer);
705
+ auto responseOffset = FBS::Consumer::CreateSetPreferredLayersResponse(
706
+ request->GetBufferBuilder(), preferredLayersOffset);
707
+
708
+ request->Accept(FBS::Response::Body::Consumer_SetPreferredLayersResponse, responseOffset);
709
+
710
+ if (IsActive() && newPreferredLayers != previousPreferredLayers)
711
+ {
712
+ this->producerStreamManager->MayChangeLayers(/*force*/ true);
713
+ }
714
+
715
+ break;
716
+ }
717
+
261
718
  case Channel::ChannelRequest::Method::CONSUMER_GET_STATS:
262
719
  {
263
720
  auto responseOffset = FillBufferStats(request->GetBufferBuilder());
@@ -470,6 +927,20 @@ namespace RTC
470
927
  this->id, FBS::Notification::Event::CONSUMER_PRODUCER_RESUME);
471
928
  }
472
929
 
930
+ void Consumer::ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc)
931
+ {
932
+ MS_TRACE();
933
+
934
+ this->producerStreamManager->ProducerRtpStream(rtpStream, mappedSsrc);
935
+ }
936
+
937
+ void Consumer::ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc)
938
+ {
939
+ MS_TRACE();
940
+
941
+ this->producerStreamManager->ProducerNewRtpStream(rtpStream, mappedSsrc);
942
+ }
943
+
473
944
  void Consumer::ProducerRtpStreamScores(const std::vector<uint8_t>* scores)
474
945
  {
475
946
  MS_TRACE();
@@ -478,6 +949,23 @@ namespace RTC
478
949
  this->producerRtpStreamScores = scores;
479
950
  }
480
951
 
952
+ void Consumer::ProducerRtpStreamScore(
953
+ RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore)
954
+ {
955
+ MS_TRACE();
956
+
957
+ EmitScore();
958
+
959
+ this->producerStreamManager->ProducerRtpStreamScore(rtpStream, score, previousScore);
960
+ }
961
+
962
+ void Consumer::ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first)
963
+ {
964
+ MS_TRACE();
965
+
966
+ this->producerStreamManager->ProducerRtcpSenderReport(rtpStream, first);
967
+ }
968
+
481
969
  // The caller (Router) is supposed to proceed with the deletion of this Consumer
482
970
  // right after calling this method. Otherwise ugly things may happen.
483
971
  void Consumer::ProducerClosed()
@@ -494,31 +982,846 @@ namespace RTC
494
982
  this->listener->OnConsumerProducerClosed(this);
495
983
  }
496
984
 
497
- void Consumer::EmitTraceEventRtpAndKeyFrameTypes(const RTC::RTP::Packet* packet, bool isRtx) const
985
+ uint8_t Consumer::GetBitratePriority() const
498
986
  {
499
987
  MS_TRACE();
500
988
 
501
- if (this->traceEventTypes.keyframe && packet->IsKeyFrame())
502
- {
503
- auto rtpPacketDump = packet->FillBuffer(this->shared->GetChannelNotifier()->GetBufferBuilder());
504
- auto traceInfo = FBS::Consumer::CreateKeyFrameTraceInfo(
505
- this->shared->GetChannelNotifier()->GetBufferBuilder(), rtpPacketDump, isRtx);
506
-
507
- auto notification = FBS::Consumer::CreateTraceNotification(
508
- this->shared->GetChannelNotifier()->GetBufferBuilder(),
509
- FBS::Consumer::TraceEventType::KEYFRAME,
510
- this->shared->GetTimeMs(),
511
- FBS::Common::TraceDirection::DIRECTION_OUT,
512
- FBS::Consumer::TraceInfo::KeyFrameTraceInfo,
513
- traceInfo.Union());
989
+ MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed");
514
990
 
515
- EmitTraceEvent(notification);
516
- }
517
- else if (this->traceEventTypes.rtp)
991
+ // Audio does not play the BWE game.
992
+ if (this->kind != RTC::Media::Kind::VIDEO)
518
993
  {
519
- auto rtpPacketDump = packet->FillBuffer(this->shared->GetChannelNotifier()->GetBufferBuilder());
520
- auto traceInfo = FBS::Consumer::CreateRtpTraceInfo(
521
- this->shared->GetChannelNotifier()->GetBufferBuilder(), rtpPacketDump, isRtx);
994
+ return 0u;
995
+ }
996
+
997
+ if (!IsActive())
998
+ {
999
+ return 0u;
1000
+ }
1001
+
1002
+ return this->priority;
1003
+ }
1004
+
1005
+ uint32_t Consumer::IncreaseLayer(uint32_t bitrate, bool considerLoss)
1006
+ {
1007
+ MS_TRACE();
1008
+
1009
+ MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed");
1010
+ MS_ASSERT(IsActive(), "should be active");
1011
+
1012
+ // Pipe does not play the BWE game.
1013
+ if (this->pipe)
1014
+ {
1015
+ return 0u;
1016
+ }
1017
+
1018
+ float lossPercentage{ 0.0f };
1019
+
1020
+ auto* rtpStream = this->mapSsrcRtpStream.begin()->second;
1021
+
1022
+ if (considerLoss)
1023
+ {
1024
+ lossPercentage = rtpStream->GetLossPercentage();
1025
+ }
1026
+
1027
+ auto nowMs = DepLibUV::GetTimeMs();
1028
+
1029
+ return this->producerStreamManager->IncreaseLayer(bitrate, considerLoss, lossPercentage, nowMs);
1030
+ }
1031
+
1032
+ void Consumer::ApplyLayers()
1033
+ {
1034
+ MS_TRACE();
1035
+
1036
+ MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed");
1037
+ MS_ASSERT(IsActive(), "should be active");
1038
+
1039
+ // Pipe does not play the BWE game.
1040
+ if (this->pipe)
1041
+ {
1042
+ return;
1043
+ }
1044
+
1045
+ auto* rtpStream = this->mapSsrcRtpStream.begin()->second;
1046
+
1047
+ this->producerStreamManager->ApplyLayers(rtpStream->GetActiveMs());
1048
+ }
1049
+
1050
+ uint32_t Consumer::GetDesiredBitrate() const
1051
+ {
1052
+ MS_TRACE();
1053
+
1054
+ MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed");
1055
+
1056
+ // Pipe does not play the BWE game.
1057
+ if (this->pipe)
1058
+ {
1059
+ return 0u;
1060
+ }
1061
+
1062
+ // Audio does not play the BWE game.
1063
+ if (this->kind != RTC::Media::Kind::VIDEO)
1064
+ {
1065
+ return 0u;
1066
+ }
1067
+
1068
+ if (!IsActive())
1069
+ {
1070
+ return 0u;
1071
+ }
1072
+
1073
+ auto nowMs = DepLibUV::GetTimeMs();
1074
+ auto desiredBitrate = this->producerStreamManager->GetDesiredBitrate(nowMs);
1075
+
1076
+ // If consumer.rtpParameters.encodings[0].maxBitrate was given and it's
1077
+ // greater than computed one, then use it.
1078
+ auto maxBitrate = this->rtpParameters.encodings[0].maxBitrate;
1079
+
1080
+ desiredBitrate = std::max(maxBitrate, desiredBitrate);
1081
+
1082
+ return desiredBitrate;
1083
+ }
1084
+
1085
+ // NOLINTNEXTLINE(misc-no-recursion)
1086
+ void Consumer::SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket)
1087
+ {
1088
+ MS_TRACE();
1089
+
1090
+ #ifdef MS_RTC_LOGGER_RTP
1091
+ packet->logger.consumerId = this->id;
1092
+ #endif
1093
+
1094
+ RTC::RTP::RtpStreamSend* rtpStream;
1095
+ RTC::SeqManager<uint16_t>* rtpSeqManager;
1096
+ RetransmissionBuffer* targetLayerRetransmissionBuffer;
1097
+
1098
+ if (this->pipe)
1099
+ {
1100
+ auto ssrc = this->mapMappedSsrcSsrc.at(packet->GetSsrc());
1101
+ rtpStream = this->mapSsrcRtpStream.at(ssrc);
1102
+ rtpSeqManager = &this->mapRtpStreamRtpSeqManager.at(rtpStream);
1103
+ targetLayerRetransmissionBuffer =
1104
+ &this->mapRtpStreamTargetLayerRetransmissionBuffer.at(rtpStream);
1105
+ }
1106
+ else
1107
+ {
1108
+ rtpStream = this->mapSsrcRtpStream.begin()->second;
1109
+ rtpSeqManager = &this->mapRtpStreamRtpSeqManager.at(rtpStream);
1110
+ targetLayerRetransmissionBuffer =
1111
+ &this->mapRtpStreamTargetLayerRetransmissionBuffer.at(rtpStream);
1112
+ }
1113
+
1114
+ if (!IsActive())
1115
+ {
1116
+ if (this->producerStreamManager->IsPacketForCurrentStream(packet))
1117
+ {
1118
+ #ifdef MS_RTC_LOGGER_RTP
1119
+ packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::CONSUMER_INACTIVE);
1120
+ #endif
1121
+ rtpSeqManager->Drop(packet->GetSequenceNumber());
1122
+ }
1123
+
1124
+ return;
1125
+ }
1126
+
1127
+ auto payloadType = packet->GetPayloadType();
1128
+
1129
+ // NOTE: This may happen if this Consumer supports just some codecs of those
1130
+ // in the corresponding Producer.
1131
+ if (!this->supportedCodecPayloadTypes[payloadType])
1132
+ {
1133
+ if (this->producerStreamManager->IsPacketForCurrentStream(packet))
1134
+ {
1135
+ MS_WARN_DEV("payload type not supported [payloadType:%" PRIu8 "]", payloadType);
1136
+
1137
+ #ifdef MS_RTC_LOGGER_RTP
1138
+ packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::UNSUPPORTED_PAYLOAD_TYPE);
1139
+ #endif
1140
+ rtpSeqManager->Drop(packet->GetSequenceNumber());
1141
+ }
1142
+
1143
+ return;
1144
+ }
1145
+
1146
+ // Ask the ProducerStreamManager to process the packet.
1147
+ auto action = this->producerStreamManager->ProcessRtpPacket(
1148
+ packet, this->lastSentPacketHasMarker, rtpStream->GetClockRate(), rtpStream->GetMaxPacketTs());
1149
+
1150
+ switch (action.type)
1151
+ {
1152
+ case ProducerStreamManager::RtpPacketProcessResult::Type::DROP:
1153
+ {
1154
+ rtpSeqManager->Drop(packet->GetSequenceNumber());
1155
+ return;
1156
+ }
1157
+
1158
+ case ProducerStreamManager::RtpPacketProcessResult::Type::SILENT_DROP:
1159
+ {
1160
+ // Don't account in seq manager.
1161
+ return;
1162
+ }
1163
+
1164
+ case ProducerStreamManager::RtpPacketProcessResult::Type::BUFFER:
1165
+ {
1166
+ StorePacketInTargetLayerRetransmissionBuffer(
1167
+ *targetLayerRetransmissionBuffer, packet, sharedPacket);
1168
+
1169
+ return;
1170
+ }
1171
+
1172
+ case ProducerStreamManager::RtpPacketProcessResult::Type::FORWARD:
1173
+ {
1174
+ // Continue below.
1175
+ break;
1176
+ }
1177
+ }
1178
+
1179
+ // Handle sync.
1180
+ if (action.isSyncPacket)
1181
+ {
1182
+ rtpSeqManager->Sync(action.syncSeqValue);
1183
+ }
1184
+
1185
+ // Handle spatial layer switch.
1186
+ if (action.spatialLayerSwitched)
1187
+ {
1188
+ // Reset the score of our RtpStream to 10.
1189
+ rtpStream->ResetScore(10u, /*notify*/ false);
1190
+
1191
+ // Emit the layersChange event.
1192
+ EmitLayersChange();
1193
+
1194
+ // Emit the score event.
1195
+ EmitScore();
1196
+ }
1197
+
1198
+ // Handle temporal layer change.
1199
+ if (action.temporalLayerChanged)
1200
+ {
1201
+ EmitLayersChange();
1202
+ }
1203
+
1204
+ // Update RTP seq number and timestamp based on offset.
1205
+ uint16_t seq;
1206
+ const uint32_t timestamp = packet->GetTimestamp() - action.tsOffset;
1207
+
1208
+ rtpSeqManager->Input(packet->GetSequenceNumber(), seq);
1209
+
1210
+ // Save original packet fields.
1211
+ auto origSsrc = packet->GetSsrc();
1212
+ auto origSeq = packet->GetSequenceNumber();
1213
+ auto origTimestamp = packet->GetTimestamp();
1214
+ const bool origMarker = packet->HasMarker();
1215
+
1216
+ // Rewrite packet.
1217
+ // For pipe each stream has its own SSRC; for non-pipe always encodings[0].
1218
+ packet->SetSsrc(this->pipe ? rtpStream->GetSsrc() : this->rtpParameters.encodings[0].ssrc);
1219
+ packet->SetSequenceNumber(seq);
1220
+ packet->SetTimestamp(timestamp);
1221
+ packet->SetMarker(action.marker);
1222
+
1223
+ #ifdef MS_RTC_LOGGER_RTP
1224
+ packet->logger.sendRtpTimestamp = timestamp;
1225
+ packet->logger.sendSeqNumber = seq;
1226
+ #endif
1227
+
1228
+ if (action.isSyncPacket)
1229
+ {
1230
+ MS_DEBUG_TAG(
1231
+ rtp,
1232
+ "sending sync packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32
1233
+ "] from original [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]",
1234
+ packet->GetSsrc(),
1235
+ packet->GetSequenceNumber(),
1236
+ packet->GetTimestamp(),
1237
+ origSsrc,
1238
+ origSeq,
1239
+ origTimestamp);
1240
+ }
1241
+
1242
+ const RTC::RTP::RtpStreamSend::ReceivePacketResult result =
1243
+ rtpStream->ReceivePacket(packet, sharedPacket);
1244
+
1245
+ if (result != RTC::RTP::RtpStreamSend::ReceivePacketResult::DISCARDED)
1246
+ {
1247
+ if (rtpSeqManager->GetMaxOutput() == packet->GetSequenceNumber())
1248
+ {
1249
+ this->lastSentPacketHasMarker = packet->HasMarker();
1250
+ }
1251
+
1252
+ // Send the packet.
1253
+ this->listener->OnConsumerSendRtpPacket(this, packet);
1254
+
1255
+ // May emit 'trace' event.
1256
+ EmitTraceEventRtpAndKeyFrameTypes(packet);
1257
+ }
1258
+ else
1259
+ {
1260
+ MS_WARN_TAG(
1261
+ rtp,
1262
+ "failed to send packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32
1263
+ "] from original [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]",
1264
+ packet->GetSsrc(),
1265
+ packet->GetSequenceNumber(),
1266
+ packet->GetTimestamp(),
1267
+ origSsrc,
1268
+ origSeq,
1269
+ origTimestamp);
1270
+
1271
+ #ifdef MS_RTC_LOGGER_RTP
1272
+ packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::SEND_RTP_STREAM_DISCARDED);
1273
+ #endif
1274
+ }
1275
+
1276
+ // Restore packet fields.
1277
+ packet->SetSsrc(origSsrc);
1278
+ packet->SetSequenceNumber(origSeq);
1279
+ packet->SetTimestamp(origTimestamp);
1280
+ packet->SetMarker(origMarker);
1281
+
1282
+ // Restore the original payload if needed.
1283
+ packet->RestorePayload();
1284
+
1285
+ // If sharedPacket doesn't have a packet inside and it has been stored we
1286
+ // need to clone the packet into it.
1287
+ if (!sharedPacket.HasPacket() && result == RTC::RTP::RtpStreamSend::ReceivePacketResult::ACCEPTED_AND_STORED)
1288
+ {
1289
+ sharedPacket.Assign(packet);
1290
+ }
1291
+
1292
+ // If sent packet was the first packet of a key frame, let's send buffered
1293
+ // packets belonging to the same key frame that arrived earlier due to
1294
+ // packet misorder.
1295
+ if (action.sendBufferedPackets)
1296
+ {
1297
+ // NOTE: Only send buffered packets if the first packet containing the
1298
+ // key frame was sent.
1299
+ if (result != RTC::RTP::RtpStreamSend::ReceivePacketResult::DISCARDED)
1300
+ {
1301
+ for (auto& kv : *targetLayerRetransmissionBuffer)
1302
+ {
1303
+ auto& bufferedSharedPacket = kv.second;
1304
+ auto* bufferedPacket = bufferedSharedPacket.GetPacket();
1305
+
1306
+ if (bufferedPacket->GetSequenceNumber() > origSeq)
1307
+ {
1308
+ MS_DEBUG_DEV(
1309
+ "sending packet buffered in the target layer retransmission buffer "
1310
+ "[ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32
1311
+ "] after sending first packet of the key frame [ssrc:%" PRIu32 ", seq:%" PRIu16
1312
+ ", ts:%" PRIu32 "]",
1313
+ bufferedPacket->GetSsrc(),
1314
+ bufferedPacket->GetSequenceNumber(),
1315
+ bufferedPacket->GetTimestamp(),
1316
+ packet->GetSsrc(),
1317
+ packet->GetSequenceNumber(),
1318
+ packet->GetTimestamp());
1319
+
1320
+ SendRtpPacket(bufferedPacket, bufferedSharedPacket);
1321
+
1322
+ // Be sure that the target layer retransmission buffer has not
1323
+ // been emptied as a result of sending this packet. If so, exit
1324
+ // the loop.
1325
+ if (targetLayerRetransmissionBuffer->empty())
1326
+ {
1327
+ MS_DEBUG_DEV(
1328
+ "target layer retransmission buffer emptied while iterating "
1329
+ "it, exiting the loop");
1330
+
1331
+ break;
1332
+ }
1333
+ }
1334
+ }
1335
+ }
1336
+
1337
+ targetLayerRetransmissionBuffer->clear();
1338
+ }
1339
+ }
1340
+
1341
+ bool Consumer::GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs)
1342
+ {
1343
+ MS_TRACE();
1344
+
1345
+ // Special condition for pipe consumer since this method will be called in a
1346
+ // loop for each stream.
1347
+ if (this->pipe)
1348
+ {
1349
+ if (
1350
+ nowMs != this->lastRtcpSentTime &&
1351
+ static_cast<float>((nowMs - this->lastRtcpSentTime) * 1.15) < this->maxRtcpInterval)
1352
+ {
1353
+ return true;
1354
+ }
1355
+ }
1356
+ else if (static_cast<float>((nowMs - this->lastRtcpSentTime) * 1.15) < this->maxRtcpInterval)
1357
+ {
1358
+ return true;
1359
+ }
1360
+
1361
+ std::vector<RTCP::SenderReport*> senderReports;
1362
+ std::vector<RTCP::SdesChunk*> sdesChunks;
1363
+ std::vector<RTCP::DelaySinceLastRr::SsrcInfo*> delaySinceLastRrSsrcInfos;
1364
+
1365
+ for (auto* rtpStream : this->rtpStreams)
1366
+ {
1367
+ auto* report = rtpStream->GetRtcpSenderReport(nowMs);
1368
+
1369
+ if (!report)
1370
+ {
1371
+ continue;
1372
+ }
1373
+
1374
+ senderReports.push_back(report);
1375
+
1376
+ // Build SDES chunk for this sender.
1377
+ auto* sdesChunk = rtpStream->GetRtcpSdesChunk();
1378
+ sdesChunks.push_back(sdesChunk);
1379
+
1380
+ auto* delaySinceLastRrSsrcInfo = rtpStream->GetRtcpXrDelaySinceLastRrSsrcInfo(nowMs);
1381
+
1382
+ if (delaySinceLastRrSsrcInfo)
1383
+ {
1384
+ delaySinceLastRrSsrcInfos.push_back(delaySinceLastRrSsrcInfo);
1385
+ }
1386
+ }
1387
+
1388
+ // RTCP Compound packet buffer cannot hold the data.
1389
+ if (!packet->Add(senderReports, sdesChunks, delaySinceLastRrSsrcInfos))
1390
+ {
1391
+ return false;
1392
+ }
1393
+
1394
+ this->lastRtcpSentTime = nowMs;
1395
+
1396
+ return true;
1397
+ }
1398
+
1399
+ void Consumer::NeedWorstRemoteFractionLost(uint32_t /*mappedSsrc*/, uint8_t& worstRemoteFractionLost)
1400
+ {
1401
+ MS_TRACE();
1402
+
1403
+ if (!IsActive())
1404
+ {
1405
+ return;
1406
+ }
1407
+
1408
+ for (auto* rtpStream : this->rtpStreams)
1409
+ {
1410
+ auto fractionLost = rtpStream->GetFractionLost();
1411
+
1412
+ // If our fraction lost is worse than the given one, update it.
1413
+ worstRemoteFractionLost = std::max(fractionLost, worstRemoteFractionLost);
1414
+ }
1415
+ }
1416
+
1417
+ void Consumer::ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket)
1418
+ {
1419
+ MS_TRACE();
1420
+
1421
+ if (!IsActive())
1422
+ {
1423
+ return;
1424
+ }
1425
+
1426
+ // May emit 'trace' event.
1427
+ EmitTraceEventNackType();
1428
+
1429
+ RTC::RTP::RtpStreamSend* rtpStream;
1430
+
1431
+ if (this->pipe)
1432
+ {
1433
+ auto ssrc = nackPacket->GetMediaSsrc();
1434
+ rtpStream = this->mapSsrcRtpStream.at(ssrc);
1435
+ }
1436
+ else
1437
+ {
1438
+ rtpStream = this->mapSsrcRtpStream.begin()->second;
1439
+ }
1440
+
1441
+ rtpStream->ReceiveNack(nackPacket);
1442
+ }
1443
+
1444
+ void Consumer::ReceiveKeyFrameRequest(RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc)
1445
+ {
1446
+ MS_TRACE();
1447
+
1448
+ if (this->kind != RTC::Media::Kind::VIDEO)
1449
+ {
1450
+ return;
1451
+ }
1452
+
1453
+ switch (messageType)
1454
+ {
1455
+ case RTC::RTCP::FeedbackPs::MessageType::PLI:
1456
+ {
1457
+ EmitTraceEventPliType(ssrc);
1458
+
1459
+ break;
1460
+ }
1461
+
1462
+ case RTC::RTCP::FeedbackPs::MessageType::FIR:
1463
+ {
1464
+ EmitTraceEventFirType(ssrc);
1465
+
1466
+ break;
1467
+ }
1468
+
1469
+ default:;
1470
+ }
1471
+
1472
+ auto* rtpStream = this->mapSsrcRtpStream.at(ssrc);
1473
+
1474
+ rtpStream->ReceiveKeyFrameRequest(messageType);
1475
+
1476
+ if (IsActive())
1477
+ {
1478
+ if (this->pipe)
1479
+ {
1480
+ for (auto& consumableRtpEncoding : this->consumableRtpEncodings)
1481
+ {
1482
+ auto mappedSsrc = consumableRtpEncoding.ssrc;
1483
+
1484
+ this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc);
1485
+ }
1486
+ }
1487
+ else
1488
+ {
1489
+ this->producerStreamManager->RequestKeyFrameForCurrentSpatialLayer();
1490
+ }
1491
+ }
1492
+ }
1493
+
1494
+ void Consumer::ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report)
1495
+ {
1496
+ MS_TRACE();
1497
+
1498
+ auto* rtpStream = this->mapSsrcRtpStream.at(report->GetSsrc());
1499
+
1500
+ rtpStream->ReceiveRtcpReceiverReport(report);
1501
+ }
1502
+
1503
+ void Consumer::ReceiveRtcpXrReceiverReferenceTime(RTC::RTCP::ReceiverReferenceTime* report)
1504
+ {
1505
+ MS_TRACE();
1506
+
1507
+ for (auto* rtpStream : this->rtpStreams)
1508
+ {
1509
+ rtpStream->ReceiveRtcpXrReceiverReferenceTime(report);
1510
+ }
1511
+ }
1512
+
1513
+ uint32_t Consumer::GetTransmissionRate(uint64_t nowMs)
1514
+ {
1515
+ MS_TRACE();
1516
+
1517
+ if (!IsActive())
1518
+ {
1519
+ return 0u;
1520
+ }
1521
+
1522
+ uint32_t rate{ 0u };
1523
+
1524
+ for (auto* rtpStream : this->rtpStreams)
1525
+ {
1526
+ rate += rtpStream->GetBitrate(nowMs);
1527
+ }
1528
+
1529
+ return rate;
1530
+ }
1531
+
1532
+ float Consumer::GetRtt() const
1533
+ {
1534
+ MS_TRACE();
1535
+
1536
+ float rtt{ 0 };
1537
+
1538
+ for (auto* rtpStream : this->rtpStreams)
1539
+ {
1540
+ rtt = std::max(rtpStream->GetRtt(), rtt);
1541
+ }
1542
+
1543
+ return rtt;
1544
+ }
1545
+
1546
+ void Consumer::UserOnTransportConnected()
1547
+ {
1548
+ MS_TRACE();
1549
+
1550
+ this->producerStreamManager->OnTransportConnected();
1551
+ }
1552
+
1553
+ void Consumer::UserOnTransportDisconnected()
1554
+ {
1555
+ MS_TRACE();
1556
+
1557
+ for (auto* rtpStream : this->rtpStreams)
1558
+ {
1559
+ rtpStream->Pause();
1560
+ }
1561
+
1562
+ for (auto& kv : this->mapRtpStreamTargetLayerRetransmissionBuffer)
1563
+ {
1564
+ auto& targetLayerRetransmissionBuffer = kv.second;
1565
+
1566
+ targetLayerRetransmissionBuffer.clear();
1567
+ }
1568
+
1569
+ this->producerStreamManager->OnTransportDisconnected();
1570
+ }
1571
+
1572
+ void Consumer::UserOnPaused()
1573
+ {
1574
+ MS_TRACE();
1575
+
1576
+ for (auto* rtpStream : this->rtpStreams)
1577
+ {
1578
+ rtpStream->Pause();
1579
+ }
1580
+
1581
+ for (auto& kv : this->mapRtpStreamTargetLayerRetransmissionBuffer)
1582
+ {
1583
+ auto& targetLayerRetransmissionBuffer = kv.second;
1584
+
1585
+ targetLayerRetransmissionBuffer.clear();
1586
+ }
1587
+
1588
+ this->producerStreamManager->OnPaused();
1589
+
1590
+ if (this->externallyManagedBitrate)
1591
+ {
1592
+ // Audio does not play the BWE game.
1593
+ if (this->kind != RTC::Media::Kind::VIDEO)
1594
+ {
1595
+ return;
1596
+ }
1597
+
1598
+ this->listener->OnConsumerNeedZeroBitrate(this);
1599
+ }
1600
+ }
1601
+
1602
+ void Consumer::UserOnResumed()
1603
+ {
1604
+ MS_TRACE();
1605
+
1606
+ this->producerStreamManager->OnResumed();
1607
+ }
1608
+
1609
+ void Consumer::CreateRtpStreams()
1610
+ {
1611
+ MS_TRACE();
1612
+
1613
+ // NOTE: For non-pipe consumers, all spatial layers are multiplexed through
1614
+ // a single RtpStreamSend (encodings[0]). Pipe consumers need one stream per
1615
+ // encoding. Here we know that SSRCs in Consumer's rtpParameters must be the
1616
+ // same as in the given consumableRtpEncodings.
1617
+ const size_t numStreams = this->pipe ? this->rtpParameters.encodings.size() : 1u;
1618
+
1619
+ for (size_t idx{ 0u }; idx < numStreams; ++idx)
1620
+ {
1621
+ auto& encoding = this->rtpParameters.encodings[idx];
1622
+ const auto* mediaCodec = this->rtpParameters.GetCodecForEncoding(encoding);
1623
+ auto& consumableEncoding = this->consumableRtpEncodings[idx];
1624
+
1625
+ MS_DEBUG_TAG(
1626
+ rtp, "[ssrc:%" PRIu32 ", payloadType:%" PRIu8 "]", encoding.ssrc, mediaCodec->payloadType);
1627
+
1628
+ // Set stream params.
1629
+ RTC::RTP::RtpStream::Params params;
1630
+
1631
+ params.encodingIdx = idx;
1632
+ params.ssrc = encoding.ssrc;
1633
+ params.payloadType = mediaCodec->payloadType;
1634
+ params.mimeType = mediaCodec->mimeType;
1635
+ params.clockRate = mediaCodec->clockRate;
1636
+ params.cname = this->rtpParameters.rtcp.cname;
1637
+ params.spatialLayers = encoding.spatialLayers;
1638
+ params.temporalLayers = encoding.temporalLayers;
1639
+
1640
+ // Check in band FEC in codec parameters.
1641
+ if (mediaCodec->parameters.HasInteger("useinbandfec") && mediaCodec->parameters.GetInteger("useinbandfec") == 1)
1642
+ {
1643
+ MS_DEBUG_TAG(rtp, "in band FEC enabled");
1644
+
1645
+ params.useInBandFec = true;
1646
+ }
1647
+
1648
+ // Check DTX in codec parameters.
1649
+ if (mediaCodec->parameters.HasInteger("usedtx") && mediaCodec->parameters.GetInteger("usedtx") == 1)
1650
+ {
1651
+ MS_DEBUG_TAG(rtp, "DTX enabled");
1652
+
1653
+ params.useDtx = true;
1654
+ }
1655
+
1656
+ // Check DTX in the encoding.
1657
+ if (encoding.dtx)
1658
+ {
1659
+ MS_DEBUG_TAG(rtp, "DTX enabled");
1660
+
1661
+ params.useDtx = true;
1662
+ }
1663
+
1664
+ for (const auto& fb : mediaCodec->rtcpFeedback)
1665
+ {
1666
+ if (!params.useNack && fb.type == "nack" && fb.parameter.empty())
1667
+ {
1668
+ MS_DEBUG_2TAGS(rtp, rtcp, "NACK supported");
1669
+
1670
+ params.useNack = true;
1671
+ }
1672
+ else if (!params.usePli && fb.type == "nack" && fb.parameter == "pli")
1673
+ {
1674
+ MS_DEBUG_2TAGS(rtp, rtcp, "PLI supported");
1675
+
1676
+ params.usePli = true;
1677
+ }
1678
+ else if (!params.useFir && fb.type == "ccm" && fb.parameter == "fir")
1679
+ {
1680
+ MS_DEBUG_2TAGS(rtp, rtcp, "FIR supported");
1681
+
1682
+ params.useFir = true;
1683
+ }
1684
+ }
1685
+
1686
+ auto* rtpStream =
1687
+ new RTC::RTP::RtpStreamSend(this, this->shared, params, this->rtpParameters.mid);
1688
+
1689
+ // If the Consumer is paused, tell the RtpStreamSend.
1690
+ if (IsPaused() || IsProducerPaused())
1691
+ {
1692
+ rtpStream->Pause();
1693
+ }
1694
+
1695
+ const auto* rtxCodec = this->rtpParameters.GetRtxCodecForEncoding(encoding);
1696
+
1697
+ if (rtxCodec && encoding.hasRtx)
1698
+ {
1699
+ rtpStream->SetRtx(rtxCodec->payloadType, encoding.rtx.ssrc);
1700
+ }
1701
+
1702
+ this->rtpStreams.push_back(rtpStream);
1703
+ this->mapMappedSsrcSsrc[consumableEncoding.ssrc] = encoding.ssrc;
1704
+ this->mapSsrcRtpStream[encoding.ssrc] = rtpStream;
1705
+
1706
+ // Let's choose an initial output seq number between 1000 and 32768 to avoid
1707
+ // libsrtp bug:
1708
+ // https://github.com/versatica/mediasoup/issues/1437
1709
+ const auto initialOutputSeq =
1710
+ Utils::Crypto::GetRandomUInt<uint16_t>(1000u, std::numeric_limits<uint16_t>::max() / 2);
1711
+
1712
+ this->mapRtpStreamRtpSeqManager[rtpStream] = RTC::SeqManager<uint16_t>(initialOutputSeq);
1713
+
1714
+ this->mapRtpStreamTargetLayerRetransmissionBuffer[rtpStream];
1715
+ }
1716
+ }
1717
+
1718
+ void Consumer::StorePacketInTargetLayerRetransmissionBuffer(
1719
+ std::map<uint16_t, RTC::RTP::SharedPacket, RTC::SeqManager<uint16_t>::SeqLowerThan>&
1720
+ targetLayerRetransmissionBuffer,
1721
+ RTC::RTP::Packet* packet,
1722
+ RTC::RTP::SharedPacket& sharedPacket)
1723
+ {
1724
+ MS_TRACE();
1725
+
1726
+ MS_DEBUG_DEV(
1727
+ "storing packet in target layer retransmission buffer [ssrc:%" PRIu32 ", seq:%" PRIu16
1728
+ ", ts:%" PRIu32 "]",
1729
+ packet->GetSsrc(),
1730
+ packet->GetSequenceNumber(),
1731
+ packet->GetTimestamp());
1732
+
1733
+ // Store original packet into the buffer. Only clone once and only if
1734
+ // necessary.
1735
+ if (!sharedPacket.HasPacket())
1736
+ {
1737
+ sharedPacket.Assign(packet);
1738
+ }
1739
+ // Assert that, if sharedPacket was already filled, both packet and
1740
+ // sharedPacket are the very same RTP packet.
1741
+ else
1742
+ {
1743
+ sharedPacket.AssertSamePacket(packet);
1744
+ }
1745
+
1746
+ targetLayerRetransmissionBuffer[packet->GetSequenceNumber()] = sharedPacket;
1747
+
1748
+ if (targetLayerRetransmissionBuffer.size() > TargetLayerRetransmissionBufferSize)
1749
+ {
1750
+ targetLayerRetransmissionBuffer.erase(targetLayerRetransmissionBuffer.begin());
1751
+ }
1752
+ }
1753
+
1754
+ void Consumer::EmitScore() const
1755
+ {
1756
+ MS_TRACE();
1757
+
1758
+ auto scoreOffset = FillBufferScore(this->shared->GetChannelNotifier()->GetBufferBuilder());
1759
+
1760
+ auto notificationOffset = FBS::Consumer::CreateScoreNotification(
1761
+ this->shared->GetChannelNotifier()->GetBufferBuilder(), scoreOffset);
1762
+
1763
+ this->shared->GetChannelNotifier()->Emit(
1764
+ this->id,
1765
+ FBS::Notification::Event::CONSUMER_SCORE,
1766
+ FBS::Notification::Body::Consumer_ScoreNotification,
1767
+ notificationOffset);
1768
+ }
1769
+
1770
+ void Consumer::EmitLayersChange() const
1771
+ {
1772
+ MS_TRACE();
1773
+
1774
+ MS_DEBUG_DEV(
1775
+ "current layers changed to [spatial:%" PRIi16 ", temporal:%" PRIi16 ", consumerId:%s]",
1776
+ this->producerStreamManager->GetCurrentSpatialLayer(),
1777
+ this->producerStreamManager->GetCurrentTemporalLayer(),
1778
+ this->id.c_str());
1779
+
1780
+ flatbuffers::Offset<FBS::Consumer::ConsumerLayers> layersOffset;
1781
+
1782
+ if (this->producerStreamManager->GetCurrentSpatialLayer() >= 0)
1783
+ {
1784
+ layersOffset = FBS::Consumer::CreateConsumerLayers(
1785
+ this->shared->GetChannelNotifier()->GetBufferBuilder(),
1786
+ this->producerStreamManager->GetCurrentSpatialLayer(),
1787
+ this->producerStreamManager->GetCurrentTemporalLayer());
1788
+ }
1789
+
1790
+ auto notificationOffset = FBS::Consumer::CreateLayersChangeNotification(
1791
+ this->shared->GetChannelNotifier()->GetBufferBuilder(), layersOffset);
1792
+
1793
+ this->shared->GetChannelNotifier()->Emit(
1794
+ this->id,
1795
+ FBS::Notification::Event::CONSUMER_LAYERS_CHANGE,
1796
+ FBS::Notification::Body::Consumer_LayersChangeNotification,
1797
+ notificationOffset);
1798
+ }
1799
+
1800
+ void Consumer::EmitTraceEventRtpAndKeyFrameTypes(const RTC::RTP::Packet* packet, bool isRtx) const
1801
+ {
1802
+ MS_TRACE();
1803
+
1804
+ if (this->traceEventTypes.keyframe && packet->IsKeyFrame())
1805
+ {
1806
+ auto rtpPacketDump = packet->FillBuffer(this->shared->GetChannelNotifier()->GetBufferBuilder());
1807
+ auto traceInfo = FBS::Consumer::CreateKeyFrameTraceInfo(
1808
+ this->shared->GetChannelNotifier()->GetBufferBuilder(), rtpPacketDump, isRtx);
1809
+
1810
+ auto notification = FBS::Consumer::CreateTraceNotification(
1811
+ this->shared->GetChannelNotifier()->GetBufferBuilder(),
1812
+ FBS::Consumer::TraceEventType::KEYFRAME,
1813
+ this->shared->GetTimeMs(),
1814
+ FBS::Common::TraceDirection::DIRECTION_OUT,
1815
+ FBS::Consumer::TraceInfo::KeyFrameTraceInfo,
1816
+ traceInfo.Union());
1817
+
1818
+ EmitTraceEvent(notification);
1819
+ }
1820
+ else if (this->traceEventTypes.rtp)
1821
+ {
1822
+ auto rtpPacketDump = packet->FillBuffer(this->shared->GetChannelNotifier()->GetBufferBuilder());
1823
+ auto traceInfo = FBS::Consumer::CreateRtpTraceInfo(
1824
+ this->shared->GetChannelNotifier()->GetBufferBuilder(), rtpPacketDump, isRtx);
522
1825
 
523
1826
  auto notification = FBS::Consumer::CreateTraceNotification(
524
1827
  this->shared->GetChannelNotifier()->GetBufferBuilder(),
@@ -606,4 +1909,75 @@ namespace RTC
606
1909
  FBS::Notification::Body::Consumer_TraceNotification,
607
1910
  notification);
608
1911
  }
1912
+
1913
+ void Consumer::OnRtpStreamScore(
1914
+ RTC::RTP::RtpStream* /*rtpStream*/, uint8_t /*score*/, uint8_t /*previousScore*/)
1915
+ {
1916
+ MS_TRACE();
1917
+
1918
+ // Emit the score event.
1919
+ EmitScore();
1920
+
1921
+ // NOTE @jmillan: Previously present in Simulcast and SVC. Does it make sense to move it here?
1922
+ // Previously present in Simulcast and SVC. Does it make sense to move it here?
1923
+ if (IsActive())
1924
+ {
1925
+ // Just check target layers if our bitrate is not externally managed.
1926
+ if (!this->externallyManagedBitrate)
1927
+ {
1928
+ this->producerStreamManager->MayChangeLayers(/*force*/ false);
1929
+ }
1930
+ }
1931
+ }
1932
+
1933
+ void Consumer::OnRtpStreamRetransmitRtpPacket(
1934
+ RTC::RTP::RtpStreamSend* rtpStream, RTC::RTP::Packet* packet)
1935
+ {
1936
+ MS_TRACE();
1937
+
1938
+ this->listener->OnConsumerRetransmitRtpPacket(this, packet);
1939
+
1940
+ // May emit 'trace' event.
1941
+ EmitTraceEventRtpAndKeyFrameTypes(packet, rtpStream->HasRtx());
1942
+ }
1943
+
1944
+ /* ProducerStreamManager::Listener methods. */
1945
+
1946
+ void Consumer::OnProducerStreamManagerKeyFrameRequested(uint32_t mappedSsrc)
1947
+ {
1948
+ MS_TRACE();
1949
+
1950
+ this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc);
1951
+ }
1952
+
1953
+ void Consumer::OnProducerStreamManagerNeedBitrateChange()
1954
+ {
1955
+ MS_TRACE();
1956
+
1957
+ this->listener->OnConsumerNeedBitrateChange(this);
1958
+ }
1959
+
1960
+ void Consumer::OnProducerStreamManagerLayersChanged()
1961
+ {
1962
+ MS_TRACE();
1963
+
1964
+ EmitLayersChange();
1965
+ }
1966
+
1967
+ void Consumer::OnProducerStreamManagerClearRetransmissionBuffer()
1968
+ {
1969
+ MS_TRACE();
1970
+
1971
+ for (auto& kv : this->mapRtpStreamTargetLayerRetransmissionBuffer)
1972
+ {
1973
+ kv.second.clear();
1974
+ }
1975
+ }
1976
+
1977
+ void Consumer::OnProducerStreamManagerScore()
1978
+ {
1979
+ MS_TRACE();
1980
+
1981
+ EmitScore();
1982
+ }
609
1983
  } // namespace RTC