@observertc/observer-js 1.0.0-beta.13 → 1.0.0-beta.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +643 -61
- package/dist/index.d.mts +2731 -74
- package/dist/index.d.ts +2731 -74
- package/dist/index.js +3471 -374
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +3421 -373
- package/dist/index.mjs.map +1 -1
- package/package.json +14 -4
- package/llms-full.txt +0 -1632
package/dist/index.js
CHANGED
|
@@ -30,15 +30,27 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
30
30
|
// src/index.ts
|
|
31
31
|
var src_exports = {};
|
|
32
32
|
__export(src_exports, {
|
|
33
|
+
ActiveIssuesRegistry: () => ActiveIssuesRegistry,
|
|
34
|
+
CODEC_MISMATCH_ISSUE: () => CODEC_MISMATCH_ISSUE,
|
|
35
|
+
CallConcurrentIssueDetector: () => CallConcurrentIssueDetector,
|
|
36
|
+
CallConcurrentIssueTypes: () => CallConcurrentIssueTypes,
|
|
37
|
+
CallHealthAggregator: () => CallHealthAggregator,
|
|
33
38
|
ClientEventTypes: () => ClientEventTypes,
|
|
34
39
|
ClientMetaTypes: () => ClientMetaTypes,
|
|
40
|
+
ClientPopulationIssueDetector: () => ClientPopulationIssueDetector,
|
|
41
|
+
ClientPopulationIssueTypes: () => ClientPopulationIssueTypes,
|
|
35
42
|
ClientSampleSink: () => ClientSampleSink,
|
|
43
|
+
CodecConsistencyValidator: () => CodecConsistencyValidator,
|
|
36
44
|
Detectors: () => Detectors,
|
|
37
45
|
InMemorySink: () => InMemorySink,
|
|
46
|
+
IssueFanOutDetector: () => IssueFanOutDetector,
|
|
47
|
+
IssueFanOutTypes: () => IssueFanOutTypes,
|
|
38
48
|
JsonlFileSink: () => JsonlFileSink,
|
|
49
|
+
LOWEST_COMMON_DENOMINATOR_ISSUE: () => LOWEST_COMMON_DENOMINATOR_ISSUE,
|
|
39
50
|
ObservedCall: () => ObservedCall,
|
|
40
51
|
ObservedCertificate: () => ObservedCertificate,
|
|
41
52
|
ObservedClient: () => ObservedClient,
|
|
53
|
+
ObservedClientIssueRegistry: () => ObservedClientIssueRegistry,
|
|
42
54
|
ObservedCodec: () => ObservedCodec,
|
|
43
55
|
ObservedDataChannel: () => ObservedDataChannel,
|
|
44
56
|
ObservedIceCandidate: () => ObservedIceCandidate,
|
|
@@ -56,14 +68,51 @@ __export(src_exports, {
|
|
|
56
68
|
ObservedRemoteInboundRtp: () => ObservedRemoteInboundRtp,
|
|
57
69
|
ObservedRemoteOutboundRtp: () => ObservedRemoteOutboundRtp,
|
|
58
70
|
Observer: () => Observer,
|
|
71
|
+
ObserverConcurrentIssueDetector: () => ObserverConcurrentIssueDetector,
|
|
72
|
+
ObserverConcurrentIssueTypes: () => ObserverConcurrentIssueTypes,
|
|
73
|
+
PublisherFaultCorroborationDetector: () => PublisherFaultCorroborationDetector,
|
|
74
|
+
PublisherFaultTypes: () => PublisherFaultTypes,
|
|
75
|
+
RESOLVED_ISSUE_SUFFIX: () => RESOLVED_ISSUE_SUFFIX,
|
|
59
76
|
RemoteTrackResolver: () => RemoteTrackResolver,
|
|
77
|
+
RemoteTrackResolverValidator: () => RemoteTrackResolverValidator,
|
|
78
|
+
SfuCongestionDetector: () => SfuCongestionDetector,
|
|
79
|
+
SimulcastReceiverValidator: () => SimulcastReceiverValidator,
|
|
80
|
+
SlidingWindow: () => SlidingWindow,
|
|
81
|
+
TrackDeliveryMismatchDetector: () => TrackDeliveryMismatchDetector,
|
|
82
|
+
TrackDeliveryMismatchTypes: () => TrackDeliveryMismatchTypes,
|
|
83
|
+
TrendTester: () => TrendTester,
|
|
84
|
+
TurnServerHealthDetector: () => TurnServerHealthDetector,
|
|
85
|
+
TurnServerHealthTypes: () => TurnServerHealthTypes,
|
|
86
|
+
TurnServerOutageDetector: () => TurnServerOutageDetector,
|
|
87
|
+
TurnServerOutageTypes: () => TurnServerOutageTypes,
|
|
88
|
+
UNRESOLVED_TRACK_LINKS_ISSUE: () => UNRESOLVED_TRACK_LINKS_ISSUE,
|
|
89
|
+
UnconsumedTrackDetector: () => UnconsumedTrackDetector,
|
|
90
|
+
UnconsumedTrackTypes: () => UnconsumedTrackTypes,
|
|
91
|
+
baseIssueType: () => baseIssueType,
|
|
92
|
+
concludeCallIssue: () => concludeCallIssue,
|
|
93
|
+
concludeObserverIssue: () => concludeObserverIssue,
|
|
94
|
+
correlation: () => correlation,
|
|
95
|
+
counterDelta: () => counterDelta,
|
|
60
96
|
createDefaultMediasoupRemoteTrackResolverFactory: () => createDefaultMediasoupRemoteTrackResolverFactory,
|
|
61
97
|
createInMemorySink: () => createInMemorySink,
|
|
62
98
|
createJsonlFileSink: () => createJsonlFileSink,
|
|
63
99
|
createJsonlFileSinkFactory: () => createJsonlFileSinkFactory,
|
|
64
100
|
createLogger: () => createLogger,
|
|
65
101
|
createP2pRemoteTrackResolverFactory: () => createP2pRemoteTrackResolverFactory,
|
|
66
|
-
|
|
102
|
+
defaultClientHealthThresholds: () => defaultClientHealthThresholds,
|
|
103
|
+
isClientIssueResolutionEntry: () => isClientIssueResolutionEntry,
|
|
104
|
+
issuePayloadAsString: () => issuePayloadAsString,
|
|
105
|
+
issuePayloadOf: () => issuePayloadOf,
|
|
106
|
+
mannKendall: () => mannKendall,
|
|
107
|
+
mannKendallVerdict: () => mannKendallVerdict,
|
|
108
|
+
median: () => median,
|
|
109
|
+
medianAbsoluteDeviation: () => medianAbsoluteDeviation,
|
|
110
|
+
pageHinkley: () => pageHinkley,
|
|
111
|
+
percentile: () => percentile,
|
|
112
|
+
percentileOfSorted: () => percentileOfSorted,
|
|
113
|
+
robustZScore: () => robustZScore,
|
|
114
|
+
setObserverLogger: () => setObserverLogger,
|
|
115
|
+
summarize: () => summarize
|
|
67
116
|
});
|
|
68
117
|
module.exports = __toCommonJS(src_exports);
|
|
69
118
|
|
|
@@ -106,8 +155,8 @@ function createLogger(moduleName) {
|
|
|
106
155
|
}
|
|
107
156
|
}();
|
|
108
157
|
}
|
|
109
|
-
function setObserverLogger(
|
|
110
|
-
mainLogger =
|
|
158
|
+
function setObserverLogger(logger9) {
|
|
159
|
+
mainLogger = logger9;
|
|
111
160
|
}
|
|
112
161
|
|
|
113
162
|
// src/ObservedCall.ts
|
|
@@ -119,6 +168,145 @@ var import_events2 = require("events");
|
|
|
119
168
|
// src/ObservedPeerConnection.ts
|
|
120
169
|
var import_events = require("events");
|
|
121
170
|
|
|
171
|
+
// src/utils/stats.ts
|
|
172
|
+
function percentile(values, p) {
|
|
173
|
+
if (values.length === 0) return void 0;
|
|
174
|
+
return percentileOfSorted(values.slice().sort(ascending), p);
|
|
175
|
+
}
|
|
176
|
+
function median(values) {
|
|
177
|
+
return percentile(values, 0.5);
|
|
178
|
+
}
|
|
179
|
+
function medianAbsoluteDeviation(values) {
|
|
180
|
+
const centre = median(values);
|
|
181
|
+
if (centre === void 0) return void 0;
|
|
182
|
+
return median(values.map((value) => Math.abs(value - centre)));
|
|
183
|
+
}
|
|
184
|
+
function robustZScore(value, baseline) {
|
|
185
|
+
const centre = median(baseline);
|
|
186
|
+
if (centre === void 0) return void 0;
|
|
187
|
+
const mad = medianAbsoluteDeviation(baseline) ?? 0;
|
|
188
|
+
if (mad === 0) return centre < value ? Infinity : 0;
|
|
189
|
+
return (value - centre) / (1.4826 * mad);
|
|
190
|
+
}
|
|
191
|
+
function summarize(values) {
|
|
192
|
+
const count = values.length;
|
|
193
|
+
if (count === 0) return void 0;
|
|
194
|
+
const sorted = values.slice().sort(ascending);
|
|
195
|
+
let sum = 0;
|
|
196
|
+
for (let i = 0; i < count; i++) sum += sorted[i];
|
|
197
|
+
return {
|
|
198
|
+
count,
|
|
199
|
+
min: sorted[0],
|
|
200
|
+
max: sorted[count - 1],
|
|
201
|
+
mean: sum / count,
|
|
202
|
+
p25: percentileOfSorted(sorted, 0.25),
|
|
203
|
+
median: percentileOfSorted(sorted, 0.5),
|
|
204
|
+
p75: percentileOfSorted(sorted, 0.75),
|
|
205
|
+
p95: percentileOfSorted(sorted, 0.95)
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
var ascending = (a, b) => a - b;
|
|
209
|
+
function percentileOfSorted(sorted, p) {
|
|
210
|
+
if (sorted.length === 1) return sorted[0];
|
|
211
|
+
const clamped = Math.min(1, Math.max(0, p));
|
|
212
|
+
const position = clamped * (sorted.length - 1);
|
|
213
|
+
const lowerIndex = Math.floor(position);
|
|
214
|
+
const upperIndex = Math.ceil(position);
|
|
215
|
+
if (lowerIndex === upperIndex) return sorted[lowerIndex];
|
|
216
|
+
const weight = position - lowerIndex;
|
|
217
|
+
return sorted[lowerIndex] * (1 - weight) + sorted[upperIndex] * weight;
|
|
218
|
+
}
|
|
219
|
+
function counterDelta(previous, current) {
|
|
220
|
+
if (previous === void 0 || current === void 0) return 0;
|
|
221
|
+
if (current < previous) return 0;
|
|
222
|
+
return current - previous;
|
|
223
|
+
}
|
|
224
|
+
function correlation(xs, ys) {
|
|
225
|
+
const n = Math.min(xs.length, ys.length);
|
|
226
|
+
if (n < 2) return 0;
|
|
227
|
+
let meanX = 0;
|
|
228
|
+
let meanY = 0;
|
|
229
|
+
for (let i = 0; i < n; i++) {
|
|
230
|
+
meanX += xs[i];
|
|
231
|
+
meanY += ys[i];
|
|
232
|
+
}
|
|
233
|
+
meanX /= n;
|
|
234
|
+
meanY /= n;
|
|
235
|
+
let covariance = 0;
|
|
236
|
+
let varianceX = 0;
|
|
237
|
+
let varianceY = 0;
|
|
238
|
+
for (let i = 0; i < n; i++) {
|
|
239
|
+
const dx = xs[i] - meanX;
|
|
240
|
+
const dy = ys[i] - meanY;
|
|
241
|
+
covariance += dx * dy;
|
|
242
|
+
varianceX += dx * dx;
|
|
243
|
+
varianceY += dy * dy;
|
|
244
|
+
}
|
|
245
|
+
if (varianceX === 0 || varianceY === 0) return 0;
|
|
246
|
+
return Math.max(0, covariance / Math.sqrt(varianceX * varianceY));
|
|
247
|
+
}
|
|
248
|
+
function pageHinkley(values, delta = 0, lambda = 50) {
|
|
249
|
+
const statistic = [];
|
|
250
|
+
let changePointIndex;
|
|
251
|
+
let sum = 0;
|
|
252
|
+
let cumulative = 0;
|
|
253
|
+
let runningMinimum = 0;
|
|
254
|
+
for (let i = 0; i < values.length; i++) {
|
|
255
|
+
sum += values[i];
|
|
256
|
+
const mean = sum / (i + 1);
|
|
257
|
+
cumulative += values[i] - mean - delta;
|
|
258
|
+
if (cumulative < runningMinimum) runningMinimum = cumulative;
|
|
259
|
+
const ph = cumulative - runningMinimum;
|
|
260
|
+
statistic.push(ph);
|
|
261
|
+
if (changePointIndex === void 0 && lambda < ph) changePointIndex = i;
|
|
262
|
+
}
|
|
263
|
+
return { statistic, changePointIndex, changeDetected: changePointIndex !== void 0 };
|
|
264
|
+
}
|
|
265
|
+
function mannKendall(values, alpha = 0.05) {
|
|
266
|
+
const n = values.length;
|
|
267
|
+
if (n < 2) return { s: 0, variance: 0, z: 0, pValue: 1, trend: "no-trend" };
|
|
268
|
+
let s = 0;
|
|
269
|
+
for (let i = 0; i < n - 1; i++) {
|
|
270
|
+
for (let j = i + 1; j < n; j++) {
|
|
271
|
+
s += Math.sign(values[j] - values[i]);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
const tieGroupSizes = /* @__PURE__ */ new Map();
|
|
275
|
+
for (const value of values) tieGroupSizes.set(value, (tieGroupSizes.get(value) ?? 0) + 1);
|
|
276
|
+
let tieCorrection = 0;
|
|
277
|
+
for (const groupSize of tieGroupSizes.values()) {
|
|
278
|
+
if (1 < groupSize) tieCorrection += groupSize * (groupSize - 1) * (2 * groupSize + 5);
|
|
279
|
+
}
|
|
280
|
+
const variance = (n * (n - 1) * (2 * n + 5) - tieCorrection) / 18;
|
|
281
|
+
return { s, variance, ...mannKendallVerdict(s, variance, alpha) };
|
|
282
|
+
}
|
|
283
|
+
function mannKendallVerdict(s, variance, alpha = 0.05) {
|
|
284
|
+
let z = 0;
|
|
285
|
+
if (variance > 0) {
|
|
286
|
+
if (s > 0) z = (s - 1) / Math.sqrt(variance);
|
|
287
|
+
else if (s < 0) z = (s + 1) / Math.sqrt(variance);
|
|
288
|
+
}
|
|
289
|
+
const pValue = 2 * (1 - standardNormalCdf(Math.abs(z)));
|
|
290
|
+
const trend = z !== 0 && pValue < alpha ? z > 0 ? "increasing" : "decreasing" : "no-trend";
|
|
291
|
+
return { z, pValue, trend };
|
|
292
|
+
}
|
|
293
|
+
function standardNormalCdf(x) {
|
|
294
|
+
return 0.5 * (1 + erf(x / Math.SQRT2));
|
|
295
|
+
}
|
|
296
|
+
function erf(x) {
|
|
297
|
+
const sign = x < 0 ? -1 : 1;
|
|
298
|
+
const ax = Math.abs(x);
|
|
299
|
+
const a1 = 0.254829592;
|
|
300
|
+
const a2 = -0.284496736;
|
|
301
|
+
const a3 = 1.421413741;
|
|
302
|
+
const a4 = -1.453152027;
|
|
303
|
+
const a5 = 1.061405429;
|
|
304
|
+
const p = 0.3275911;
|
|
305
|
+
const t = 1 / (1 + p * ax);
|
|
306
|
+
const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-(ax * ax));
|
|
307
|
+
return sign * y;
|
|
308
|
+
}
|
|
309
|
+
|
|
122
310
|
// src/ObservedInboundRtp.ts
|
|
123
311
|
var ObservedInboundRtp = class {
|
|
124
312
|
constructor(timestamp, id, ssrc, kind, trackIdentifier, _peerConnection) {
|
|
@@ -206,6 +394,48 @@ var ObservedInboundRtp = class {
|
|
|
206
394
|
deltaBytesReceived = 0;
|
|
207
395
|
deltaReceivedSamples = 0;
|
|
208
396
|
deltaSilentConcealedSamples = 0;
|
|
397
|
+
// Per-tick, counter-reset-safe deltas of the cumulative RTCStats counters. These are what
|
|
398
|
+
// detectors need ("how many freezes/PLIs in THIS tick"), as opposed to the lifetime totals.
|
|
399
|
+
deltaConcealedSamples = 0;
|
|
400
|
+
deltaConcealmentEvents = 0;
|
|
401
|
+
deltaFreezeCount = 0;
|
|
402
|
+
deltaFreezesDuration = 0;
|
|
403
|
+
deltaPliCount = 0;
|
|
404
|
+
deltaNackCount = 0;
|
|
405
|
+
deltaFirCount = 0;
|
|
406
|
+
deltaPacketsDiscarded = 0;
|
|
407
|
+
deltaFramesDecoded = 0;
|
|
408
|
+
deltaFramesReceived = 0;
|
|
409
|
+
deltaFramesRendered = 0;
|
|
410
|
+
deltaFramesDropped = 0;
|
|
411
|
+
deltaKeyFramesDecoded = 0;
|
|
412
|
+
deltaDecodeTime = 0;
|
|
413
|
+
deltaJitterBufferDelay = 0;
|
|
414
|
+
deltaJitterBufferEmittedCount = 0;
|
|
415
|
+
deltaRetransmittedPacketsReceived = 0;
|
|
416
|
+
deltaFecPacketsReceived = 0;
|
|
417
|
+
deltaFecPacketsDiscarded = 0;
|
|
418
|
+
deltaPausesDuration = 0;
|
|
419
|
+
/**
|
|
420
|
+
* Mean jitter-buffer delay for the frames/samples emitted in this tick (seconds), derived from
|
|
421
|
+
* the cumulative `jitterBufferDelay` / `jitterBufferEmittedCount` pair — the only correct way to
|
|
422
|
+
* read those two counters.
|
|
423
|
+
*/
|
|
424
|
+
jitterBufferDelayInMs;
|
|
425
|
+
/** Fraction of the samples received in this tick that were concealed (0..1). */
|
|
426
|
+
concealmentRatio;
|
|
427
|
+
/** Fraction of the frames received in this tick that were dropped before rendering (0..1). */
|
|
428
|
+
framesDroppedRatio;
|
|
429
|
+
/**
|
|
430
|
+
* `true` when the codec or decoder implementation changed in this tick.
|
|
431
|
+
*
|
|
432
|
+
* Chrome resets `packetsReceived`/`bytesReceived` on an SSRC when the codec switches
|
|
433
|
+
* (crbug.com/webrtc/5361, open since 2015), which shows up as a sawtooth spike or a negative
|
|
434
|
+
* rate. Every delta in this tick is therefore suppressed to `0` rather than reported as traffic
|
|
435
|
+
* — otherwise a room-wide codec rollout produces a synchronized fake-degradation alert across
|
|
436
|
+
* every participant at once.
|
|
437
|
+
*/
|
|
438
|
+
counterResetBoundary = false;
|
|
209
439
|
// Derived from the corresponding remote-outbound-rtp (sender report), when present.
|
|
210
440
|
remoteRttInMs;
|
|
211
441
|
remoteBytesSent;
|
|
@@ -246,31 +476,76 @@ var ObservedInboundRtp = class {
|
|
|
246
476
|
this.remotePacketsSent = void 0;
|
|
247
477
|
this.remoteTimestamp = void 0;
|
|
248
478
|
this.bitrate = 0;
|
|
249
|
-
this.jitter = void 0;
|
|
250
479
|
this.fractionLost = void 0;
|
|
251
480
|
this.bitPerPixel = 0;
|
|
481
|
+
this.deltaConcealedSamples = 0;
|
|
482
|
+
this.deltaConcealmentEvents = 0;
|
|
483
|
+
this.deltaFreezeCount = 0;
|
|
484
|
+
this.deltaFreezesDuration = 0;
|
|
485
|
+
this.deltaPausesDuration = 0;
|
|
486
|
+
this.deltaPliCount = 0;
|
|
487
|
+
this.deltaNackCount = 0;
|
|
488
|
+
this.deltaFirCount = 0;
|
|
489
|
+
this.deltaPacketsDiscarded = 0;
|
|
490
|
+
this.deltaFramesDecoded = 0;
|
|
491
|
+
this.deltaFramesReceived = 0;
|
|
492
|
+
this.deltaFramesRendered = 0;
|
|
493
|
+
this.deltaFramesDropped = 0;
|
|
494
|
+
this.deltaKeyFramesDecoded = 0;
|
|
495
|
+
this.deltaDecodeTime = 0;
|
|
496
|
+
this.deltaJitterBufferDelay = 0;
|
|
497
|
+
this.deltaJitterBufferEmittedCount = 0;
|
|
498
|
+
this.deltaRetransmittedPacketsReceived = 0;
|
|
499
|
+
this.deltaFecPacketsReceived = 0;
|
|
500
|
+
this.deltaFecPacketsDiscarded = 0;
|
|
501
|
+
this.jitterBufferDelayInMs = void 0;
|
|
502
|
+
this.concealmentRatio = void 0;
|
|
503
|
+
this.framesDroppedRatio = void 0;
|
|
252
504
|
const elapsedTimeInMs = stats.timestamp - this.timestamp;
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
505
|
+
this.counterResetBoundary = this.codecId !== void 0 && stats.codecId !== void 0 && this.codecId !== stats.codecId || this.decoderImplementation !== void 0 && stats.decoderImplementation !== void 0 && this.decoderImplementation !== stats.decoderImplementation;
|
|
506
|
+
if (0 < elapsedTimeInMs && !this.counterResetBoundary) {
|
|
507
|
+
const elapsedInSeconds = elapsedTimeInMs / 1e3;
|
|
508
|
+
this.deltaBytesReceived = counterDelta(this.bytesReceived, stats.bytesReceived);
|
|
509
|
+
this.deltaLostPackets = counterDelta(this.packetsLost, stats.packetsLost);
|
|
510
|
+
this.deltaReceivedPackets = counterDelta(this.packetsReceived, stats.packetsReceived);
|
|
511
|
+
this.deltaReceivedSamples = counterDelta(this.totalSamplesReceived, stats.totalSamplesReceived);
|
|
512
|
+
this.deltaSilentConcealedSamples = counterDelta(this.silentConcealedSamples, stats.silentConcealedSamples);
|
|
513
|
+
this.deltaConcealedSamples = counterDelta(this.concealedSamples, stats.concealedSamples);
|
|
514
|
+
this.deltaConcealmentEvents = counterDelta(this.concealmentEvents, stats.concealmentEvents);
|
|
515
|
+
this.deltaFreezeCount = counterDelta(this.freezeCount, stats.freezeCount);
|
|
516
|
+
this.deltaFreezesDuration = counterDelta(this.totalFreezesDuration, stats.totalFreezesDuration);
|
|
517
|
+
this.deltaPausesDuration = counterDelta(this.totalPausesDuration, stats.totalPausesDuration);
|
|
518
|
+
this.deltaPliCount = counterDelta(this.pliCount, stats.pliCount);
|
|
519
|
+
this.deltaNackCount = counterDelta(this.nackCount, stats.nackCount);
|
|
520
|
+
this.deltaFirCount = counterDelta(this.firCount, stats.firCount);
|
|
521
|
+
this.deltaPacketsDiscarded = counterDelta(this.packetsDiscarded, stats.packetsDiscarded);
|
|
522
|
+
this.deltaFramesDecoded = counterDelta(this.framesDecoded, stats.framesDecoded);
|
|
523
|
+
this.deltaFramesReceived = counterDelta(this.framesReceived, stats.framesReceived);
|
|
524
|
+
this.deltaFramesRendered = counterDelta(this.framesRendered, stats.framesRendered);
|
|
525
|
+
this.deltaFramesDropped = counterDelta(this.framesDropped, stats.framesDropped);
|
|
526
|
+
this.deltaKeyFramesDecoded = counterDelta(this.keyFramesDecoded, stats.keyFramesDecoded);
|
|
527
|
+
this.deltaDecodeTime = counterDelta(this.totalDecodeTime, stats.totalDecodeTime);
|
|
528
|
+
this.deltaJitterBufferDelay = counterDelta(this.jitterBufferDelay, stats.jitterBufferDelay);
|
|
529
|
+
this.deltaJitterBufferEmittedCount = counterDelta(this.jitterBufferEmittedCount, stats.jitterBufferEmittedCount);
|
|
530
|
+
this.deltaRetransmittedPacketsReceived = counterDelta(this.retransmittedPacketsReceived, stats.retransmittedPacketsReceived);
|
|
531
|
+
this.deltaFecPacketsReceived = counterDelta(this.fecPacketsReceived, stats.fecPacketsReceived);
|
|
532
|
+
this.deltaFecPacketsDiscarded = counterDelta(this.fecPacketsDiscarded, stats.fecPacketsDiscarded);
|
|
533
|
+
this.bitrate = this.deltaBytesReceived * 8 / elapsedInSeconds;
|
|
534
|
+
const pixels = (stats.frameWidth ?? 0) * (stats.frameHeight ?? 0);
|
|
535
|
+
if (0 < pixels && 0 < this.deltaFramesReceived) {
|
|
536
|
+
this.bitPerPixel = this.deltaBytesReceived * 8 / (this.deltaFramesReceived * pixels);
|
|
259
537
|
}
|
|
260
|
-
if (
|
|
261
|
-
this.
|
|
262
|
-
}
|
|
263
|
-
if (this.totalSamplesReceived && stats.totalSamplesReceived && this.totalSamplesReceived < stats.totalSamplesReceived) {
|
|
264
|
-
this.deltaReceivedSamples = stats.totalSamplesReceived - this.totalSamplesReceived;
|
|
538
|
+
if (0 < this.deltaReceivedPackets || 0 < this.deltaLostPackets) {
|
|
539
|
+
this.fractionLost = this.deltaLostPackets / (this.deltaLostPackets + this.deltaReceivedPackets);
|
|
265
540
|
}
|
|
266
|
-
if (
|
|
267
|
-
this.
|
|
541
|
+
if (0 < this.deltaJitterBufferEmittedCount) {
|
|
542
|
+
this.jitterBufferDelayInMs = this.deltaJitterBufferDelay / this.deltaJitterBufferEmittedCount * 1e3;
|
|
268
543
|
}
|
|
269
|
-
if (
|
|
270
|
-
this.
|
|
544
|
+
if (0 < this.deltaReceivedSamples) {
|
|
545
|
+
this.concealmentRatio = this.deltaConcealedSamples / this.deltaReceivedSamples;
|
|
271
546
|
}
|
|
272
|
-
if (
|
|
273
|
-
this.
|
|
547
|
+
if (0 < this.deltaFramesReceived) {
|
|
548
|
+
this.framesDroppedRatio = this.deltaFramesDropped / this.deltaFramesReceived;
|
|
274
549
|
}
|
|
275
550
|
}
|
|
276
551
|
this.timestamp = stats.timestamp;
|
|
@@ -315,6 +590,10 @@ var ObservedInboundRtp = class {
|
|
|
315
590
|
this.concealedSamples = stats.concealedSamples;
|
|
316
591
|
this.silentConcealedSamples = stats.silentConcealedSamples;
|
|
317
592
|
this.concealmentEvents = stats.concealmentEvents;
|
|
593
|
+
this.jitter = stats.jitter;
|
|
594
|
+
this.framesReceived = stats.framesReceived;
|
|
595
|
+
this.retransmittedPacketsReceived = stats.retransmittedPacketsReceived;
|
|
596
|
+
this.retransmittedBytesReceived = stats.retransmittedBytesReceived;
|
|
318
597
|
}
|
|
319
598
|
};
|
|
320
599
|
|
|
@@ -376,6 +655,25 @@ var ObservedOutboundRtp = class {
|
|
|
376
655
|
bitPerPixel = 0;
|
|
377
656
|
deltaPacketsSent = 0;
|
|
378
657
|
deltaBytesSent = 0;
|
|
658
|
+
// Per-tick, counter-reset-safe deltas of the cumulative RTCStats counters.
|
|
659
|
+
deltaFramesSent = 0;
|
|
660
|
+
deltaFramesEncoded = 0;
|
|
661
|
+
deltaKeyFramesEncoded = 0;
|
|
662
|
+
deltaNackCount = 0;
|
|
663
|
+
deltaPliCount = 0;
|
|
664
|
+
deltaFirCount = 0;
|
|
665
|
+
deltaRetransmittedPacketsSent = 0;
|
|
666
|
+
deltaRetransmittedBytesSent = 0;
|
|
667
|
+
deltaEncodeTime = 0;
|
|
668
|
+
deltaQualityLimitationResolutionChanges = 0;
|
|
669
|
+
/**
|
|
670
|
+
* `true` when the codec, encoder implementation or scalability mode changed in this tick.
|
|
671
|
+
*
|
|
672
|
+
* Chrome resets `packetsSent`/`bytesSent` on the SSRC when the codec switches
|
|
673
|
+
* (crbug.com/webrtc/5361), producing sawtooth spikes and negative bitrates. All deltas for the
|
|
674
|
+
* tick are suppressed so a codec change is never mistaken for a traffic event.
|
|
675
|
+
*/
|
|
676
|
+
counterResetBoundary = false;
|
|
379
677
|
// Derived from the corresponding remote-inbound-rtp (receiver report), when present.
|
|
380
678
|
remoteRttInMs;
|
|
381
679
|
remoteFractionLost;
|
|
@@ -409,12 +707,24 @@ var ObservedOutboundRtp = class {
|
|
|
409
707
|
this.packetRate = 0;
|
|
410
708
|
this.deltaPacketsSent = 0;
|
|
411
709
|
this.deltaBytesSent = 0;
|
|
710
|
+
this.deltaFramesSent = 0;
|
|
711
|
+
this.deltaFramesEncoded = 0;
|
|
712
|
+
this.deltaKeyFramesEncoded = 0;
|
|
713
|
+
this.deltaNackCount = 0;
|
|
714
|
+
this.deltaPliCount = 0;
|
|
715
|
+
this.deltaFirCount = 0;
|
|
716
|
+
this.deltaRetransmittedPacketsSent = 0;
|
|
717
|
+
this.deltaRetransmittedBytesSent = 0;
|
|
718
|
+
this.deltaEncodeTime = 0;
|
|
719
|
+
this.deltaQualityLimitationResolutionChanges = 0;
|
|
412
720
|
this.remoteRttInMs = void 0;
|
|
413
721
|
this.remoteFractionLost = void 0;
|
|
414
722
|
this.remoteJitter = void 0;
|
|
415
723
|
this.remotePacketsLost = void 0;
|
|
416
724
|
const elapsedTimeInMs = stats.timestamp - this.timestamp;
|
|
417
|
-
|
|
725
|
+
const changed = (a, b) => a !== void 0 && b !== void 0 && a !== b;
|
|
726
|
+
this.counterResetBoundary = changed(this.codecId, stats.codecId) || changed(this.encoderImplementation, stats.encoderImplementation) || changed(this.scalabilityMode, stats.scalabilityMode);
|
|
727
|
+
if (elapsedTimeInMs && !this.counterResetBoundary) {
|
|
418
728
|
if (stats.packetsSent !== void 0 && this.packetsSent !== void 0 && stats.packetsSent >= this.packetsSent) {
|
|
419
729
|
this.deltaPacketsSent = stats.packetsSent - this.packetsSent;
|
|
420
730
|
this.packetRate = this.deltaPacketsSent / (elapsedTimeInMs / 1e3);
|
|
@@ -423,11 +733,24 @@ var ObservedOutboundRtp = class {
|
|
|
423
733
|
this.deltaBytesSent = stats.bytesSent - this.bytesSent;
|
|
424
734
|
this.bitrate = this.deltaBytesSent * 8 / (elapsedTimeInMs / 1e3);
|
|
425
735
|
}
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
736
|
+
const deltaHeaderBytesSent = counterDelta(this.headerBytesSent, stats.headerBytesSent);
|
|
737
|
+
this.payloadBitrate = Math.max(0, this.deltaBytesSent - deltaHeaderBytesSent) * 8 / (elapsedTimeInMs / 1e3);
|
|
738
|
+
this.deltaFramesSent = counterDelta(this.framesSent, stats.framesSent);
|
|
739
|
+
this.deltaFramesEncoded = counterDelta(this.framesEncoded, stats.framesEncoded);
|
|
740
|
+
this.deltaKeyFramesEncoded = counterDelta(this.keyFramesEncoded, stats.keyFramesEncoded);
|
|
741
|
+
this.deltaNackCount = counterDelta(this.nackCount, stats.nackCount);
|
|
742
|
+
this.deltaPliCount = counterDelta(this.pliCount, stats.pliCount);
|
|
743
|
+
this.deltaFirCount = counterDelta(this.firCount, stats.firCount);
|
|
744
|
+
this.deltaRetransmittedPacketsSent = counterDelta(this.retransmittedPacketsSent, stats.retransmittedPacketsSent);
|
|
745
|
+
this.deltaRetransmittedBytesSent = counterDelta(this.retransmittedBytesSent, stats.retransmittedBytesSent);
|
|
746
|
+
this.deltaEncodeTime = counterDelta(this.totalEncodeTime, stats.totalEncodeTime);
|
|
747
|
+
this.deltaQualityLimitationResolutionChanges = counterDelta(
|
|
748
|
+
this.qualityLimitationResolutionChanges,
|
|
749
|
+
stats.qualityLimitationResolutionChanges
|
|
750
|
+
);
|
|
751
|
+
const pixels = (stats.frameWidth ?? 0) * (stats.frameHeight ?? 0);
|
|
752
|
+
if (0 < pixels && 0 < this.deltaFramesSent) {
|
|
753
|
+
this.bitPerPixel = this.deltaBytesSent * 8 / (this.deltaFramesSent * pixels);
|
|
431
754
|
}
|
|
432
755
|
}
|
|
433
756
|
this.timestamp = stats.timestamp;
|
|
@@ -1150,6 +1473,10 @@ var ObservedInboundTrack = class {
|
|
|
1150
1473
|
removedAt;
|
|
1151
1474
|
muted;
|
|
1152
1475
|
attachments;
|
|
1476
|
+
degradationReasons = [];
|
|
1477
|
+
get degraded() {
|
|
1478
|
+
return this.degradationReasons.length > 0;
|
|
1479
|
+
}
|
|
1153
1480
|
get score() {
|
|
1154
1481
|
return this.calculatedScore.value;
|
|
1155
1482
|
}
|
|
@@ -1172,9 +1499,49 @@ var ObservedInboundTrack = class {
|
|
|
1172
1499
|
this.timestamp = stats.timestamp;
|
|
1173
1500
|
this.calculatedScore.value = stats.score;
|
|
1174
1501
|
this.attachments = stats.attachments;
|
|
1502
|
+
this.checkDegradation();
|
|
1503
|
+
}
|
|
1504
|
+
checkDegradation() {
|
|
1505
|
+
const thresholds = this._peerConnection.client.call.observer.config.inboundTrackDegradationThresholds;
|
|
1506
|
+
this.degradationReasons.length = 0;
|
|
1507
|
+
if (!thresholds) return;
|
|
1508
|
+
const reasons = [];
|
|
1509
|
+
if (thresholds.deltaFreezeCount < (this._inboundRtp?.deltaFreezeCount ?? 0)) {
|
|
1510
|
+
reasons.push("freezes");
|
|
1511
|
+
}
|
|
1512
|
+
if (thresholds.framesDroppedRatio < (this._inboundRtp?.framesDroppedRatio ?? 0)) {
|
|
1513
|
+
reasons.push("frames-dropped");
|
|
1514
|
+
}
|
|
1515
|
+
if (thresholds.jitterBufferDelayInMs < (this._inboundRtp?.jitterBufferDelayInMs ?? 0)) {
|
|
1516
|
+
reasons.push("jitter-buffer-delay");
|
|
1517
|
+
}
|
|
1518
|
+
if (thresholds.concealmentRatio < (this._inboundRtp?.concealmentRatio ?? 0)) {
|
|
1519
|
+
reasons.push("concealment");
|
|
1520
|
+
}
|
|
1521
|
+
if (thresholds.rttInMs < (this._inboundRtp?.remoteRttInMs ?? 0)) {
|
|
1522
|
+
reasons.push("rtt");
|
|
1523
|
+
}
|
|
1524
|
+
this.degradationReasons = reasons;
|
|
1175
1525
|
}
|
|
1176
1526
|
};
|
|
1177
1527
|
|
|
1528
|
+
// src/common/utils.ts
|
|
1529
|
+
function pushFinite(target, value) {
|
|
1530
|
+
if (typeof value === "number" && Number.isFinite(value)) target.push(value);
|
|
1531
|
+
}
|
|
1532
|
+
function parseJsonAs(json) {
|
|
1533
|
+
if (!json) return void 0;
|
|
1534
|
+
try {
|
|
1535
|
+
return JSON.parse(json);
|
|
1536
|
+
} catch {
|
|
1537
|
+
return void 0;
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
function parseJsonObject(json) {
|
|
1541
|
+
const parsed = parseJsonAs(json);
|
|
1542
|
+
return typeof parsed === "object" && parsed !== null ? parsed : void 0;
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1178
1545
|
// src/ObservedOutboundTrack.ts
|
|
1179
1546
|
var ObservedOutboundTrack = class {
|
|
1180
1547
|
constructor(timestamp, id, kind, _peerConnection, _outboundRtps, _mediaSource) {
|
|
@@ -1194,6 +1561,10 @@ var ObservedOutboundTrack = class {
|
|
|
1194
1561
|
_visited = false;
|
|
1195
1562
|
appData;
|
|
1196
1563
|
remoteInboundTracks = /* @__PURE__ */ new Set();
|
|
1564
|
+
detectors = new Detectors();
|
|
1565
|
+
// only set if the track is being received by at least one subscriber, otherwise undefined
|
|
1566
|
+
// which means the remote track resolver has to be set up to know if the track is unconsumed or not
|
|
1567
|
+
receivingDistribution;
|
|
1197
1568
|
calculatedScore = {
|
|
1198
1569
|
weight: 1,
|
|
1199
1570
|
value: void 0
|
|
@@ -1202,6 +1573,12 @@ var ObservedOutboundTrack = class {
|
|
|
1202
1573
|
removedAt;
|
|
1203
1574
|
muted;
|
|
1204
1575
|
attachments;
|
|
1576
|
+
degradedReasons;
|
|
1577
|
+
bitrate;
|
|
1578
|
+
deltaPacketsSent;
|
|
1579
|
+
remoteFractionLost;
|
|
1580
|
+
remoteRttInMs;
|
|
1581
|
+
qualityLimitationReason;
|
|
1205
1582
|
get score() {
|
|
1206
1583
|
return this.calculatedScore.value;
|
|
1207
1584
|
}
|
|
@@ -1210,6 +1587,9 @@ var ObservedOutboundTrack = class {
|
|
|
1210
1587
|
this._visited = false;
|
|
1211
1588
|
return visited;
|
|
1212
1589
|
}
|
|
1590
|
+
get degraded() {
|
|
1591
|
+
return (this.degradedReasons?.length ?? 0) > 0;
|
|
1592
|
+
}
|
|
1213
1593
|
getPeerConnection() {
|
|
1214
1594
|
return this._peerConnection;
|
|
1215
1595
|
}
|
|
@@ -1224,30 +1604,89 @@ var ObservedOutboundTrack = class {
|
|
|
1224
1604
|
this.timestamp = stats.timestamp;
|
|
1225
1605
|
this.calculatedScore.value = stats.score;
|
|
1226
1606
|
this.attachments = stats.attachments;
|
|
1607
|
+
this.bitrate = void 0;
|
|
1608
|
+
this.deltaPacketsSent = void 0;
|
|
1609
|
+
this.remoteFractionLost = void 0;
|
|
1610
|
+
this.remoteRttInMs = void 0;
|
|
1611
|
+
this.qualityLimitationReason = void 0;
|
|
1612
|
+
for (const rtp of this._outboundRtps ?? []) {
|
|
1613
|
+
this.bitrate = (this.bitrate ?? 0) + rtp.bitrate;
|
|
1614
|
+
this.deltaPacketsSent = (this.deltaPacketsSent ?? 0) + rtp.deltaPacketsSent;
|
|
1615
|
+
if (rtp.remoteFractionLost !== void 0) {
|
|
1616
|
+
this.remoteFractionLost = Math.max(this.remoteFractionLost ?? 0, rtp.remoteFractionLost);
|
|
1617
|
+
}
|
|
1618
|
+
if (rtp.remoteRttInMs !== void 0) {
|
|
1619
|
+
this.remoteRttInMs = Math.max(this.remoteRttInMs ?? 0, rtp.remoteRttInMs);
|
|
1620
|
+
}
|
|
1621
|
+
if (rtp.qualityLimitationReason && rtp.qualityLimitationReason !== "none") {
|
|
1622
|
+
this.qualityLimitationReason = rtp.qualityLimitationReason;
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
1625
|
+
const thresholds = this._peerConnection.client.call.observer.config.outboundTrackDegradationThresholds;
|
|
1626
|
+
const reasons = [];
|
|
1627
|
+
if (thresholds && thresholds.fractionLost < (this.remoteFractionLost ?? 0)) reasons.push("remote-fraction-lost");
|
|
1628
|
+
if (thresholds && thresholds.rttInMs < (this.remoteRttInMs ?? 0)) reasons.push("remote-rtt");
|
|
1629
|
+
if (this.qualityLimitationReason) reasons.push(`quality-limited-${this.qualityLimitationReason}`);
|
|
1630
|
+
if (this.muted !== true && this.deltaPacketsSent === 0) reasons.push("no-packets-sent");
|
|
1631
|
+
this.degradedReasons = reasons.length === 0 ? void 0 : reasons;
|
|
1632
|
+
this.receivingDistribution = this.createReceivingDistribution();
|
|
1633
|
+
this.detectors.update();
|
|
1227
1634
|
}
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1635
|
+
createReceivingDistribution() {
|
|
1636
|
+
if (this.remoteInboundTracks.size === 0) return void 0;
|
|
1637
|
+
let freezeAffected = 0;
|
|
1638
|
+
let freezeTotal = 0;
|
|
1639
|
+
let pliAffected = 0;
|
|
1640
|
+
let pliTotal = 0;
|
|
1641
|
+
let concealmentAffected = 0;
|
|
1642
|
+
let numberOfReceivers = 0;
|
|
1643
|
+
let numberOfDegradedReceivers = 0;
|
|
1644
|
+
let numberOfHealthyReceivers = 0;
|
|
1645
|
+
const bitrates = [];
|
|
1646
|
+
const fractionLosts = [];
|
|
1647
|
+
const jitters = [];
|
|
1648
|
+
const rtts = [];
|
|
1649
|
+
const jitterBufferDelays = [];
|
|
1650
|
+
const concealmentRatios = [];
|
|
1651
|
+
for (const inboundTrack of this.remoteInboundTracks) {
|
|
1652
|
+
const inboundRtp = inboundTrack.getInboundRtp();
|
|
1653
|
+
const peerConnection = inboundTrack.getPeerConnection();
|
|
1654
|
+
if (!inboundRtp) continue;
|
|
1655
|
+
if (0 < inboundRtp.deltaFreezeCount) freezeAffected += 1;
|
|
1656
|
+
if (0 < inboundRtp.deltaPliCount) pliAffected += 1;
|
|
1657
|
+
if (0 < (inboundRtp.concealmentRatio ?? 0)) concealmentAffected += 1;
|
|
1658
|
+
if (inboundTrack.degraded) numberOfDegradedReceivers += 1;
|
|
1659
|
+
else numberOfHealthyReceivers += 1;
|
|
1660
|
+
pliTotal += inboundRtp.deltaPliCount;
|
|
1661
|
+
freezeTotal += inboundRtp.deltaFreezeCount;
|
|
1662
|
+
++numberOfReceivers;
|
|
1663
|
+
pushFinite(bitrates, inboundRtp.bitrate);
|
|
1664
|
+
pushFinite(fractionLosts, inboundRtp.fractionLost);
|
|
1665
|
+
pushFinite(jitters, inboundRtp.jitter);
|
|
1666
|
+
pushFinite(rtts, peerConnection?.currentRttInMs);
|
|
1667
|
+
pushFinite(jitterBufferDelays, inboundRtp.jitterBufferDelayInMs);
|
|
1668
|
+
pushFinite(concealmentRatios, inboundRtp.concealmentRatio);
|
|
1669
|
+
}
|
|
1670
|
+
return {
|
|
1671
|
+
numberOfReceivers,
|
|
1672
|
+
numberOfHealthyReceivers,
|
|
1673
|
+
numberOfDegradedReceivers,
|
|
1674
|
+
degradedRatio: numberOfReceivers === 0 ? 0 : numberOfDegradedReceivers / numberOfReceivers,
|
|
1675
|
+
bitrate: summarize(bitrates),
|
|
1676
|
+
fractionLost: summarize(fractionLosts),
|
|
1677
|
+
jitter: summarize(jitters),
|
|
1678
|
+
rttInMs: summarize(rtts),
|
|
1679
|
+
jitterBufferDelayInMs: summarize(jitterBufferDelays),
|
|
1680
|
+
concealmentRatio: summarize(concealmentRatios),
|
|
1681
|
+
freezes: { affectedReceivers: freezeAffected, total: freezeTotal },
|
|
1682
|
+
plis: { affectedReceivers: pliAffected, total: pliTotal },
|
|
1683
|
+
concealment: { affectedReceivers: concealmentAffected }
|
|
1684
|
+
};
|
|
1247
1685
|
}
|
|
1248
|
-
}
|
|
1686
|
+
};
|
|
1249
1687
|
|
|
1250
1688
|
// src/ObservedPeerConnection.ts
|
|
1689
|
+
var ascending2 = (a, b) => a - b;
|
|
1251
1690
|
var logger = createLogger("ObservedPeerConnection");
|
|
1252
1691
|
var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
1253
1692
|
constructor(peerConnectionId, client) {
|
|
@@ -1321,7 +1760,34 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1321
1760
|
sendingVideoBitrate = 0;
|
|
1322
1761
|
receivingAudioBitrate = 0;
|
|
1323
1762
|
receivingVideoBitrate = 0;
|
|
1763
|
+
/**
|
|
1764
|
+
* Median round-trip time of the tick, in ms.
|
|
1765
|
+
*
|
|
1766
|
+
* Prefers {@link rtcpRttInMs} and falls back to {@link iceRttInMs}, so within one tick it always
|
|
1767
|
+
* reports **one** kind of round trip. It used to be the median of both mixed together, which was
|
|
1768
|
+
* a bug: the mixing ratio changed as streams came and went, so the value moved for reasons that
|
|
1769
|
+
* had nothing to do with the network.
|
|
1770
|
+
*/
|
|
1324
1771
|
currentRttInMs;
|
|
1772
|
+
/**
|
|
1773
|
+
* RTT measured by ICE/STUN consent checks, in ms — the trip to **whatever terminates ICE**. In an
|
|
1774
|
+
* SFU topology that is the SFU, so this is the client↔SFU leg, not client↔client.
|
|
1775
|
+
*/
|
|
1776
|
+
iceRttInMs;
|
|
1777
|
+
/**
|
|
1778
|
+
* RTT reported by RTCP receiver reports, in ms — an **end-to-end** media-path round trip.
|
|
1779
|
+
*
|
|
1780
|
+
* Not the same trip as {@link iceRttInMs}; the difference between the two is roughly the far side
|
|
1781
|
+
* of the SFU (see {@link sfuHopRttInMs}).
|
|
1782
|
+
*/
|
|
1783
|
+
rtcpRttInMs;
|
|
1784
|
+
/**
|
|
1785
|
+
* `rtcpRttInMs - iceRttInMs`, when both are known — an estimate of everything *past* the SFU.
|
|
1786
|
+
*
|
|
1787
|
+
* Useful for splitting "this client's own last mile is slow" (high `iceRttInMs`) from "the path
|
|
1788
|
+
* beyond the SFU is slow" (low ICE, high hop).
|
|
1789
|
+
*/
|
|
1790
|
+
sfuHopRttInMs;
|
|
1325
1791
|
currentJitter;
|
|
1326
1792
|
usingTCP = false;
|
|
1327
1793
|
usingTURN = false;
|
|
@@ -1393,9 +1859,10 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1393
1859
|
return this.iceTransports.map((iceTransport) => iceTransport.getSelectedCandidatePair()).filter((pair) => pair !== void 0);
|
|
1394
1860
|
}
|
|
1395
1861
|
get selectedIceCandiadtePairForTurn() {
|
|
1396
|
-
return this.selectedIceCandidatePairs.filter(
|
|
1397
|
-
|
|
1398
|
-
|
|
1862
|
+
return this.selectedIceCandidatePairs.filter((pair) => {
|
|
1863
|
+
const localCandidate = pair.getLocalCandidate();
|
|
1864
|
+
return localCandidate?.candidateType === "relay" && localCandidate?.url?.startsWith("turn") === true;
|
|
1865
|
+
});
|
|
1399
1866
|
}
|
|
1400
1867
|
close() {
|
|
1401
1868
|
if (this.closed) return;
|
|
@@ -1471,7 +1938,8 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1471
1938
|
const now = Date.now();
|
|
1472
1939
|
const elapsedTimeInMs = now - this.updated;
|
|
1473
1940
|
const elapsedTimeInSec = elapsedTimeInMs / 1e3;
|
|
1474
|
-
const
|
|
1941
|
+
const iceRttMeasurementsInSec = [];
|
|
1942
|
+
const rtcpRttMeasurementsInSec = [];
|
|
1475
1943
|
const jitterMeasurements = [];
|
|
1476
1944
|
if (sample.certificates) {
|
|
1477
1945
|
for (const certificate of sample.certificates) {
|
|
@@ -1503,7 +1971,7 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1503
1971
|
const observedCandidatePair = this._updateIceCandidatePairStats(iceCandidatePair);
|
|
1504
1972
|
if (!observedCandidatePair) continue;
|
|
1505
1973
|
if (observedCandidatePair.currentRoundTripTime) {
|
|
1506
|
-
|
|
1974
|
+
iceRttMeasurementsInSec.push(observedCandidatePair.currentRoundTripTime);
|
|
1507
1975
|
}
|
|
1508
1976
|
if (observedCandidatePair.availableIncomingBitrate) {
|
|
1509
1977
|
this.availableIncomingBitrate += observedCandidatePair.availableIncomingBitrate;
|
|
@@ -1580,7 +2048,7 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1580
2048
|
const observedRemoteInboundRtp = this._updateRemoteInboundRtpStats(remoteInboundRtp);
|
|
1581
2049
|
if (!observedRemoteInboundRtp) continue;
|
|
1582
2050
|
if (observedRemoteInboundRtp.roundTripTime) {
|
|
1583
|
-
|
|
2051
|
+
rtcpRttMeasurementsInSec.push(observedRemoteInboundRtp.roundTripTime);
|
|
1584
2052
|
}
|
|
1585
2053
|
const observedOutboundRtp = observedRemoteInboundRtp.getOutboundRtp();
|
|
1586
2054
|
if (observedOutboundRtp) {
|
|
@@ -1651,13 +2119,12 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1651
2119
|
this.sendingVideoBitrate = this.deltaSentVideoBytes * 8 / elapsedTimeInSec;
|
|
1652
2120
|
this.receivingAudioBitrate = this.deltaReceivedAudioBytes * 8 / elapsedTimeInSec;
|
|
1653
2121
|
this.receivingVideoBitrate = this.deltaReceivedVideoBytes * 8 / elapsedTimeInSec;
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
}
|
|
2122
|
+
this.iceRttInMs = 0 < iceRttMeasurementsInSec.length ? percentileOfSorted(iceRttMeasurementsInSec.sort(ascending2), 0.5) * 1e3 : void 0;
|
|
2123
|
+
this.rtcpRttInMs = 0 < rtcpRttMeasurementsInSec.length ? percentileOfSorted(rtcpRttMeasurementsInSec.sort(ascending2), 0.5) * 1e3 : void 0;
|
|
2124
|
+
this.currentRttInMs = this.rtcpRttInMs ?? this.iceRttInMs;
|
|
2125
|
+
this.sfuHopRttInMs = this.rtcpRttInMs !== void 0 && this.iceRttInMs !== void 0 ? Math.max(0, this.rtcpRttInMs - this.iceRttInMs) : void 0;
|
|
1659
2126
|
if (jitterMeasurements.length > 0) {
|
|
1660
|
-
this.currentJitter =
|
|
2127
|
+
this.currentJitter = percentileOfSorted(jitterMeasurements.sort(ascending2), 0.5);
|
|
1661
2128
|
} else {
|
|
1662
2129
|
this.currentJitter = void 0;
|
|
1663
2130
|
}
|
|
@@ -1667,12 +2134,15 @@ var ObservedPeerConnection = class extends import_events.EventEmitter {
|
|
|
1667
2134
|
this.usingTCP = false;
|
|
1668
2135
|
this.usingTURN = false;
|
|
1669
2136
|
for (const selectedCandidatePair of selectedIceCandidatePairs) {
|
|
1670
|
-
|
|
2137
|
+
const localCandidate = selectedCandidatePair.getLocalCandidate();
|
|
2138
|
+
if (localCandidate?.protocol === "tcp") {
|
|
1671
2139
|
this.usingTCP = true;
|
|
1672
2140
|
}
|
|
1673
|
-
if (
|
|
1674
|
-
selectedCandidatePairForTurn.push(selectedCandidatePair);
|
|
2141
|
+
if (localCandidate?.candidateType === "relay") {
|
|
1675
2142
|
this.usingTURN = true;
|
|
2143
|
+
if (localCandidate.url?.startsWith("turn")) {
|
|
2144
|
+
selectedCandidatePairForTurn.push(selectedCandidatePair);
|
|
2145
|
+
}
|
|
1676
2146
|
}
|
|
1677
2147
|
this.deltaTransportReceivedBytes += selectedCandidatePair.deltaBytesReceived;
|
|
1678
2148
|
this.deltaTransportSentBytes += selectedCandidatePair.deltaBytesSent;
|
|
@@ -2164,10 +2634,19 @@ var ClientMetaTypes = /* @__PURE__ */ ((ClientMetaTypes2) => {
|
|
|
2164
2634
|
return ClientMetaTypes2;
|
|
2165
2635
|
})(ClientMetaTypes || {});
|
|
2166
2636
|
|
|
2637
|
+
// src/issues/ActiveClientIssue.ts
|
|
2638
|
+
var RESOLVED_ISSUE_SUFFIX = "-resolved";
|
|
2639
|
+
function isClientIssueResolutionEntry(issue) {
|
|
2640
|
+
return issue.type.endsWith(RESOLVED_ISSUE_SUFFIX);
|
|
2641
|
+
}
|
|
2642
|
+
function baseIssueType(type) {
|
|
2643
|
+
return type.endsWith(RESOLVED_ISSUE_SUFFIX) ? type.slice(0, -RESOLVED_ISSUE_SUFFIX.length) : type;
|
|
2644
|
+
}
|
|
2645
|
+
|
|
2167
2646
|
// src/ObservedClient.ts
|
|
2168
2647
|
var logger2 = createLogger("ObservedClient");
|
|
2169
2648
|
var ObservedClient = class extends import_events2.EventEmitter {
|
|
2170
|
-
constructor(settings, call) {
|
|
2649
|
+
constructor(settings, call, activeIssues) {
|
|
2171
2650
|
super();
|
|
2172
2651
|
this.call = call;
|
|
2173
2652
|
this.setMaxListeners(Infinity);
|
|
@@ -2183,6 +2662,7 @@ var ObservedClient = class extends import_events2.EventEmitter {
|
|
|
2183
2662
|
this.sink.once("close", () => this.sink?.off("error", onError));
|
|
2184
2663
|
this.sink.on("error", onError);
|
|
2185
2664
|
}
|
|
2665
|
+
this.activeIssues = activeIssues;
|
|
2186
2666
|
}
|
|
2187
2667
|
call;
|
|
2188
2668
|
clientId;
|
|
@@ -2249,7 +2729,16 @@ var ObservedClient = class extends import_events2.EventEmitter {
|
|
|
2249
2729
|
numberOfScoreMeasurements = 0;
|
|
2250
2730
|
// public totalNumberOfIssues = 0;
|
|
2251
2731
|
mediaDevices = [];
|
|
2252
|
-
|
|
2732
|
+
/**
|
|
2733
|
+
* The client's currently **open** stateful issues, keyed by `ClientIssue.key` — the server-side
|
|
2734
|
+
* mirror of the client monitor's own active-issue map (client-monitor-js >= 4.6.0).
|
|
2735
|
+
*
|
|
2736
|
+
* This turns point-in-time symptom reports into intervals, which is what lets detectors ask
|
|
2737
|
+
* "are these clients broken *at the same time*" instead of "did they both report something
|
|
2738
|
+
* recently". Entries are opened by a raise, closed by the matching `<type>-resolved` entry, and
|
|
2739
|
+
* force-closed when the client closes.
|
|
2740
|
+
*/
|
|
2741
|
+
activeIssues;
|
|
2253
2742
|
_pendingInjections = {};
|
|
2254
2743
|
_activeSample;
|
|
2255
2744
|
closeTimer;
|
|
@@ -2262,6 +2751,10 @@ var ObservedClient = class extends import_events2.EventEmitter {
|
|
|
2262
2751
|
close() {
|
|
2263
2752
|
if (this.closed) return;
|
|
2264
2753
|
this._flushPendingInjections();
|
|
2754
|
+
const activeIssues = [...this.activeIssues.values()];
|
|
2755
|
+
for (const issue of activeIssues) {
|
|
2756
|
+
this._resolveIssue({ key: issue.key, type: `${issue.type}${RESOLVED_ISSUE_SUFFIX}` }, "client-closed");
|
|
2757
|
+
}
|
|
2265
2758
|
this.closed = true;
|
|
2266
2759
|
Array.from(this.observedPeerConnections.values()).forEach((peerConnection) => peerConnection.close());
|
|
2267
2760
|
if (!this.leftAt) {
|
|
@@ -2393,10 +2886,17 @@ var ObservedClient = class extends import_events2.EventEmitter {
|
|
|
2393
2886
|
this._processClientEvent(clientEvent);
|
|
2394
2887
|
}
|
|
2395
2888
|
this.attachments = sample.attachments;
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2889
|
+
if (0 < elapsedInSeconds) {
|
|
2890
|
+
this.receivingAudioBitrate = this.deltaReceivedAudioBytes * 8 / elapsedInSeconds;
|
|
2891
|
+
this.receivingVideoBitrate = this.deltaReceivedVideoBytes * 8 / elapsedInSeconds;
|
|
2892
|
+
this.sendingAudioBitrate = this.deltaSentAudioBytes * 8 / elapsedInSeconds;
|
|
2893
|
+
this.sendingVideoBitrate = this.deltaSentVideoBytes * 8 / elapsedInSeconds;
|
|
2894
|
+
} else {
|
|
2895
|
+
this.receivingAudioBitrate = 0;
|
|
2896
|
+
this.receivingVideoBitrate = 0;
|
|
2897
|
+
this.sendingAudioBitrate = 0;
|
|
2898
|
+
this.sendingVideoBitrate = 0;
|
|
2899
|
+
}
|
|
2400
2900
|
this.currentAvgRttInMs = 0 < numberOfRttMeasurements ? sumOfRtts / numberOfRttMeasurements : void 0;
|
|
2401
2901
|
this.calculatedScore.value = sample.score;
|
|
2402
2902
|
this.lastSampleTimestamp = sample.timestamp;
|
|
@@ -2498,13 +2998,82 @@ var ObservedClient = class extends import_events2.EventEmitter {
|
|
|
2498
2998
|
}
|
|
2499
2999
|
this._notify("client-metadata", { ...this.eventScope, metaData: metadata });
|
|
2500
3000
|
}
|
|
3001
|
+
/**
|
|
3002
|
+
* Process one `clientIssues[]` entry.
|
|
3003
|
+
*
|
|
3004
|
+
* Entries come in two flavours (client-monitor-js >= 4.6.0):
|
|
3005
|
+
*
|
|
3006
|
+
* - a **raise** — opens an {@link ActiveClientIssue} under `issue.key` and emits `client-issue`;
|
|
3007
|
+
* - a **resolution** — `type` ends in `-resolved` and carries the same `key`; it closes the
|
|
3008
|
+
* matching active issue and emits `client-issue-resolved`.
|
|
3009
|
+
*
|
|
3010
|
+
* Keyless entries are one-shot: reported, never tracked. A re-raise of a key already active
|
|
3011
|
+
* refreshes the payload rather than opening a second interval.
|
|
3012
|
+
*/
|
|
2501
3013
|
addIssue(issue) {
|
|
2502
3014
|
if (this.closed) return;
|
|
3015
|
+
if (isClientIssueResolutionEntry(issue)) {
|
|
3016
|
+
return this._resolveIssue(issue);
|
|
3017
|
+
}
|
|
3018
|
+
const now = Date.now();
|
|
3019
|
+
if (issue.key) {
|
|
3020
|
+
const payload = parseJsonObject(issue.payload);
|
|
3021
|
+
const existing = this.activeIssues.get(issue.key);
|
|
3022
|
+
if (existing) {
|
|
3023
|
+
existing.payload = payload ?? existing.payload;
|
|
3024
|
+
} else {
|
|
3025
|
+
const active = {
|
|
3026
|
+
key: issue.key,
|
|
3027
|
+
type: baseIssueType(issue.type),
|
|
3028
|
+
clientId: this.clientId,
|
|
3029
|
+
callId: this.call.callId,
|
|
3030
|
+
raisedAt: issue.timestamp ?? now,
|
|
3031
|
+
observedAt: now,
|
|
3032
|
+
payload,
|
|
3033
|
+
peerConnectionId: typeof payload?.peerConnectionId === "string" ? payload.peerConnectionId : void 0,
|
|
3034
|
+
trackId: typeof payload?.trackId === "string" ? payload.trackId : void 0
|
|
3035
|
+
};
|
|
3036
|
+
this.activeIssues.add(active);
|
|
3037
|
+
}
|
|
3038
|
+
}
|
|
2503
3039
|
this._notify("client-issue", { ...this.eventScope, issue });
|
|
2504
3040
|
}
|
|
2505
3041
|
addExtensionStats(stats) {
|
|
2506
3042
|
this._notify("client-extension-stats", { ...this.eventScope, extensionStats: stats });
|
|
2507
3043
|
}
|
|
3044
|
+
/** Close the active issue a `<type>-resolved` entry refers to, and announce the finished interval. */
|
|
3045
|
+
_resolveIssue(issue, resolvedBy = "client") {
|
|
3046
|
+
const now = Date.now();
|
|
3047
|
+
const resolutionPayload = parseJsonObject(issue.payload);
|
|
3048
|
+
const type = baseIssueType(issue.type);
|
|
3049
|
+
const raisedAt = typeof resolutionPayload?.raisedAt === "number" ? resolutionPayload.raisedAt : void 0;
|
|
3050
|
+
const active = issue.key ? this.activeIssues.get(issue.key) : void 0;
|
|
3051
|
+
if (active) {
|
|
3052
|
+
this.activeIssues.remove(active.key);
|
|
3053
|
+
}
|
|
3054
|
+
const resolvedAt = issue.timestamp ?? now;
|
|
3055
|
+
const durationInMs = typeof resolutionPayload?.durationInMs === "number" ? resolutionPayload.durationInMs : Math.max(0, resolvedAt - (active?.raisedAt ?? raisedAt ?? resolvedAt));
|
|
3056
|
+
this._notify("client-issue-resolved", {
|
|
3057
|
+
...this.eventScope,
|
|
3058
|
+
resolvedIssue: {
|
|
3059
|
+
key: active?.key ?? issue.key ?? `${this.clientId}:${type}:${raisedAt ?? resolvedAt}`,
|
|
3060
|
+
type,
|
|
3061
|
+
clientId: this.clientId,
|
|
3062
|
+
callId: this.call.callId,
|
|
3063
|
+
raisedAt: active?.raisedAt ?? raisedAt ?? resolvedAt,
|
|
3064
|
+
observedAt: active?.observedAt ?? now,
|
|
3065
|
+
payload: active?.payload,
|
|
3066
|
+
peerConnectionId: active?.peerConnectionId,
|
|
3067
|
+
trackId: active?.trackId,
|
|
3068
|
+
resolvedAt,
|
|
3069
|
+
observedResolvedAt: now,
|
|
3070
|
+
durationInMs,
|
|
3071
|
+
comment: typeof resolutionPayload?.comment === "string" ? resolutionPayload.comment : void 0,
|
|
3072
|
+
resolutionPayload,
|
|
3073
|
+
resolvedBy
|
|
3074
|
+
}
|
|
3075
|
+
});
|
|
3076
|
+
}
|
|
2508
3077
|
_processClientEvent(event, postBuffer) {
|
|
2509
3078
|
switch (event.type) {
|
|
2510
3079
|
case "CLIENT_JOINED" /* CLIENT_JOINED */: {
|
|
@@ -2807,83 +3376,6 @@ var DefaultCallScoreCalculator = class {
|
|
|
2807
3376
|
}
|
|
2808
3377
|
};
|
|
2809
3378
|
|
|
2810
|
-
// src/updaters/OnAllClientCallUpdater.ts
|
|
2811
|
-
var OnAllClientCallUpdater = class {
|
|
2812
|
-
constructor(_observedCall) {
|
|
2813
|
-
this._observedCall = _observedCall;
|
|
2814
|
-
this._onNewObservedClient = this._onNewObservedClient.bind(this);
|
|
2815
|
-
this._observedCall.once("close", () => {
|
|
2816
|
-
this._observedCall.off("newclient", this._onNewObservedClient);
|
|
2817
|
-
});
|
|
2818
|
-
this._observedCall.on("newclient", this._onNewObservedClient);
|
|
2819
|
-
}
|
|
2820
|
-
_observedCall;
|
|
2821
|
-
name = "OnAllClientCallUpdater";
|
|
2822
|
-
description = "Call the update() method of the ObservedCall once all client has been updated";
|
|
2823
|
-
_updatedClients = /* @__PURE__ */ new Set();
|
|
2824
|
-
closed = false;
|
|
2825
|
-
close() {
|
|
2826
|
-
if (this.closed) return;
|
|
2827
|
-
this.closed = true;
|
|
2828
|
-
this._updatedClients.clear();
|
|
2829
|
-
}
|
|
2830
|
-
_onNewObservedClient(observedClient) {
|
|
2831
|
-
if (this.closed) return;
|
|
2832
|
-
const onUpdate = (...args) => this._onClientUpdate(observedClient, args[2]);
|
|
2833
|
-
observedClient.once("close", () => {
|
|
2834
|
-
observedClient.off("update", onUpdate);
|
|
2835
|
-
this._updatedClients.delete(observedClient.clientId);
|
|
2836
|
-
this._updateIfEveryClientUpdated();
|
|
2837
|
-
});
|
|
2838
|
-
observedClient.on("update", onUpdate);
|
|
2839
|
-
}
|
|
2840
|
-
_onClientUpdate(observedClient, context) {
|
|
2841
|
-
if (observedClient.closed) return;
|
|
2842
|
-
this._updatedClients.add(observedClient.clientId);
|
|
2843
|
-
this._updateIfEveryClientUpdated(context);
|
|
2844
|
-
}
|
|
2845
|
-
_updateIfEveryClientUpdated(context) {
|
|
2846
|
-
if (this._updatedClients.size < this._observedCall.observedClients.size) {
|
|
2847
|
-
return;
|
|
2848
|
-
} else if (this.closed) {
|
|
2849
|
-
return;
|
|
2850
|
-
}
|
|
2851
|
-
this._updatedClients.clear();
|
|
2852
|
-
this._observedCall.update(context);
|
|
2853
|
-
}
|
|
2854
|
-
};
|
|
2855
|
-
|
|
2856
|
-
// src/updaters/OnAnyClientCallUpdater.ts
|
|
2857
|
-
var OnAnyClientCallUpdater = class {
|
|
2858
|
-
constructor(_observedCall) {
|
|
2859
|
-
this._observedCall = _observedCall;
|
|
2860
|
-
this._onNewObservedClient = this._onNewObservedClient.bind(this);
|
|
2861
|
-
this._observedCall.once("close", () => {
|
|
2862
|
-
this._observedCall.off("newclient", this._onNewObservedClient);
|
|
2863
|
-
});
|
|
2864
|
-
this._observedCall.on("newclient", this._onNewObservedClient);
|
|
2865
|
-
}
|
|
2866
|
-
_observedCall;
|
|
2867
|
-
name = "OnAnyClientCallUpdater";
|
|
2868
|
-
description = "Call the update() method of the ObservedCall when any of the client has been updated";
|
|
2869
|
-
closed = false;
|
|
2870
|
-
close() {
|
|
2871
|
-
if (this.closed) return;
|
|
2872
|
-
this.closed = true;
|
|
2873
|
-
}
|
|
2874
|
-
_onNewObservedClient(observedClient) {
|
|
2875
|
-
if (this.closed) return;
|
|
2876
|
-
const onUpdate = (...args) => {
|
|
2877
|
-
if (this.closed) return;
|
|
2878
|
-
this._observedCall.update(args[2]);
|
|
2879
|
-
};
|
|
2880
|
-
observedClient.once("close", () => {
|
|
2881
|
-
observedClient.off("update", onUpdate);
|
|
2882
|
-
});
|
|
2883
|
-
observedClient.on("update", onUpdate);
|
|
2884
|
-
}
|
|
2885
|
-
};
|
|
2886
|
-
|
|
2887
3379
|
// src/detectors/Detectors.ts
|
|
2888
3380
|
var logger3 = createLogger("Detectors");
|
|
2889
3381
|
var Detectors = class {
|
|
@@ -2894,11 +3386,18 @@ var Detectors = class {
|
|
|
2894
3386
|
get listOfNames() {
|
|
2895
3387
|
return this._detectors.map((d) => d.name);
|
|
2896
3388
|
}
|
|
3389
|
+
get size() {
|
|
3390
|
+
return this._detectors.length;
|
|
3391
|
+
}
|
|
2897
3392
|
add(detector) {
|
|
2898
3393
|
this._detectors.push(detector);
|
|
2899
3394
|
}
|
|
3395
|
+
get(name) {
|
|
3396
|
+
return this._detectors.find((detector) => detector.name === name);
|
|
3397
|
+
}
|
|
2900
3398
|
remove(detector) {
|
|
2901
3399
|
this._detectors = this._detectors.filter((d) => d !== detector);
|
|
3400
|
+
this._close(detector);
|
|
2902
3401
|
}
|
|
2903
3402
|
update() {
|
|
2904
3403
|
for (const detector of this._detectors) {
|
|
@@ -2910,128 +3409,993 @@ var Detectors = class {
|
|
|
2910
3409
|
}
|
|
2911
3410
|
}
|
|
2912
3411
|
clear() {
|
|
3412
|
+
const detectors = this._detectors;
|
|
2913
3413
|
this._detectors = [];
|
|
3414
|
+
for (const detector of detectors) this._close(detector);
|
|
3415
|
+
}
|
|
3416
|
+
_close(detector) {
|
|
3417
|
+
try {
|
|
3418
|
+
detector.close?.();
|
|
3419
|
+
} catch (err) {
|
|
3420
|
+
logger3.warn(`Error closing detector ${detector?.name}`, err);
|
|
3421
|
+
}
|
|
2914
3422
|
}
|
|
2915
3423
|
};
|
|
2916
3424
|
|
|
2917
|
-
// src/
|
|
2918
|
-
var
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
this.
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
this.
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
this.
|
|
2940
|
-
|
|
3425
|
+
// src/issues/ObservedClientIssueRegistry.ts
|
|
3426
|
+
var ObservedClientIssueRegistry = class {
|
|
3427
|
+
constructor(registry) {
|
|
3428
|
+
this.registry = registry;
|
|
3429
|
+
}
|
|
3430
|
+
registry;
|
|
3431
|
+
issues = /* @__PURE__ */ new Map();
|
|
3432
|
+
get size() {
|
|
3433
|
+
return this.issues.size;
|
|
3434
|
+
}
|
|
3435
|
+
keys() {
|
|
3436
|
+
return this.issues.keys();
|
|
3437
|
+
}
|
|
3438
|
+
values() {
|
|
3439
|
+
return this.issues.values();
|
|
3440
|
+
}
|
|
3441
|
+
get(key) {
|
|
3442
|
+
return this.issues.get(key);
|
|
3443
|
+
}
|
|
3444
|
+
add(issue) {
|
|
3445
|
+
const existing = this.issues.get(issue.key);
|
|
3446
|
+
if (existing && existing !== issue) this.registry?.delete(existing);
|
|
3447
|
+
this.issues.set(issue.key, issue);
|
|
3448
|
+
this.registry?.add(issue);
|
|
3449
|
+
return this;
|
|
3450
|
+
}
|
|
3451
|
+
remove(key) {
|
|
3452
|
+
const issue = this.issues.get(key);
|
|
3453
|
+
if (!issue) return void 0;
|
|
3454
|
+
this.issues.delete(key);
|
|
3455
|
+
this.registry?.delete(issue);
|
|
3456
|
+
return issue;
|
|
3457
|
+
}
|
|
3458
|
+
clear() {
|
|
3459
|
+
for (const key of [...this.issues.keys()]) this.remove(key);
|
|
3460
|
+
}
|
|
3461
|
+
};
|
|
3462
|
+
|
|
3463
|
+
// src/detectors/UnconsumedTrackDetector.ts
|
|
3464
|
+
var UnconsumedTrackTypes = {
|
|
3465
|
+
/** A track is being published to the SFU that nobody is subscribed to — pure wasted uplink. */
|
|
3466
|
+
unconsumedPublishedTrack: "UNCONSUMED_PUBLISHED_TRACK"
|
|
3467
|
+
};
|
|
3468
|
+
var UnconsumedTrackDetector = class _UnconsumedTrackDetector {
|
|
3469
|
+
constructor(call, config = {}) {
|
|
3470
|
+
this.call = call;
|
|
3471
|
+
this.config = {
|
|
3472
|
+
cooldownMs: 3e5,
|
|
3473
|
+
minUnconsumedDurationInMs: 3e4,
|
|
3474
|
+
minBitrate: 5e4,
|
|
3475
|
+
...config
|
|
2941
3476
|
};
|
|
2942
3477
|
}
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
3478
|
+
call;
|
|
3479
|
+
static NAME = "unconsumed-track-detector";
|
|
3480
|
+
name = _UnconsumedTrackDetector.NAME;
|
|
3481
|
+
config;
|
|
3482
|
+
/** trackId -> when it was first seen sending with no subscribers. */
|
|
3483
|
+
_unconsumedSince = /* @__PURE__ */ new Map();
|
|
3484
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
3485
|
+
update() {
|
|
3486
|
+
if (!this.call.remoteTrackResolver) return;
|
|
3487
|
+
const unconsumed = this.call.unconsumedOutboundTracks;
|
|
3488
|
+
if (unconsumed.size === 0) {
|
|
3489
|
+
if (0 < this._unconsumedSince.size) this._unconsumedSince.clear();
|
|
3490
|
+
return;
|
|
3491
|
+
}
|
|
3492
|
+
const now = Date.now();
|
|
3493
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3494
|
+
for (const outboundTrack of unconsumed) {
|
|
3495
|
+
const rtps = outboundTrack.getOutboundRtps() ?? [];
|
|
3496
|
+
let bitrate = 0;
|
|
3497
|
+
let sending = false;
|
|
3498
|
+
for (let i = 0; i < rtps.length; i++) {
|
|
3499
|
+
bitrate += rtps[i].bitrate;
|
|
3500
|
+
if (0 < rtps[i].deltaPacketsSent) sending = true;
|
|
3501
|
+
}
|
|
3502
|
+
seen.add(outboundTrack.id);
|
|
3503
|
+
if (!sending || bitrate < this.config.minBitrate) {
|
|
3504
|
+
this._unconsumedSince.delete(outboundTrack.id);
|
|
3505
|
+
continue;
|
|
3506
|
+
}
|
|
3507
|
+
const since = this._unconsumedSince.get(outboundTrack.id) ?? now;
|
|
3508
|
+
this._unconsumedSince.set(outboundTrack.id, since);
|
|
3509
|
+
const unconsumedForMs = now - since;
|
|
3510
|
+
if (unconsumedForMs < this.config.minUnconsumedDurationInMs) continue;
|
|
3511
|
+
if (now - (this._lastRaisedAt.get(outboundTrack.id) ?? 0) < this.config.cooldownMs) continue;
|
|
3512
|
+
this._lastRaisedAt.set(outboundTrack.id, now);
|
|
3513
|
+
const peerConnection = outboundTrack.getPeerConnection();
|
|
3514
|
+
this.call.addIssue({
|
|
3515
|
+
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
3516
|
+
timestamp: now,
|
|
3517
|
+
payload: {
|
|
3518
|
+
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
3519
|
+
trackId: outboundTrack.id,
|
|
3520
|
+
kind: outboundTrack.kind,
|
|
3521
|
+
publisherClientId: peerConnection?.client.clientId,
|
|
3522
|
+
peerConnectionId: peerConnection?.peerConnectionId,
|
|
3523
|
+
bitrate,
|
|
3524
|
+
unconsumedForMs,
|
|
3525
|
+
// what the waste costs, roughly, if it continues
|
|
3526
|
+
wastedBytesPerSecond: bitrate / 8
|
|
3527
|
+
}
|
|
3528
|
+
});
|
|
3529
|
+
}
|
|
3530
|
+
for (const trackId of [...this._unconsumedSince.keys()]) {
|
|
3531
|
+
if (!seen.has(trackId)) this._unconsumedSince.delete(trackId);
|
|
3532
|
+
}
|
|
2979
3533
|
}
|
|
2980
|
-
|
|
2981
|
-
|
|
3534
|
+
close() {
|
|
3535
|
+
this._unconsumedSince.clear();
|
|
3536
|
+
this._lastRaisedAt.clear();
|
|
2982
3537
|
}
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
3538
|
+
};
|
|
3539
|
+
|
|
3540
|
+
// src/detectors/TrackDeliveryMismatchDetector.ts
|
|
3541
|
+
var TrackDeliveryMismatchTypes = {
|
|
3542
|
+
/**
|
|
3543
|
+
* The source is sending, but **none** of its subscribers are receiving → the media is being lost
|
|
3544
|
+
* between the publisher and the receivers. In an SFU that means the forwarding path.
|
|
3545
|
+
*/
|
|
3546
|
+
publishedTrackNotDelivered: "PUBLISHED_TRACK_NOT_DELIVERED",
|
|
3547
|
+
/**
|
|
3548
|
+
* The source is sending and most subscribers are fine, but **some** are dry → those consumers are
|
|
3549
|
+
* broken individually (in mediasoup, the usual mitigation is recreating the consumer).
|
|
3550
|
+
*/
|
|
3551
|
+
receiverTrackNotDelivered: "RECEIVER_TRACK_NOT_DELIVERED",
|
|
3552
|
+
/**
|
|
3553
|
+
* The source itself stopped producing, so its subscribers being dry is expected and **not** an
|
|
3554
|
+
* SFU fault. Reported so the other two verdicts can be trusted as *not* being this.
|
|
3555
|
+
*/
|
|
3556
|
+
publisherTrackDry: "PUBLISHER_TRACK_DRY"
|
|
3557
|
+
};
|
|
3558
|
+
var TrackDeliveryMismatchDetector = class _TrackDeliveryMismatchDetector {
|
|
3559
|
+
constructor(call, config = {}) {
|
|
3560
|
+
this.call = call;
|
|
3561
|
+
this._config = {
|
|
3562
|
+
dryOutboundIssueType: "dry-outbound-track",
|
|
3563
|
+
dryInboundIssueType: "dry-inbound-track",
|
|
3564
|
+
minReceivers: 2,
|
|
3565
|
+
allReceiversRatio: 1,
|
|
3566
|
+
cooldownMs: 6e4,
|
|
3567
|
+
...config
|
|
3568
|
+
};
|
|
3569
|
+
this.call.activeIssuesRegistry.addIssueTracker(this._config.dryOutboundIssueType, this);
|
|
3570
|
+
this.call.activeIssuesRegistry.addIssueTracker(this._config.dryInboundIssueType, this);
|
|
2987
3571
|
}
|
|
3572
|
+
call;
|
|
3573
|
+
static NAME = "track-delivery-mismatch-detector";
|
|
3574
|
+
name = _TrackDeliveryMismatchDetector.NAME;
|
|
3575
|
+
_config;
|
|
3576
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
3577
|
+
dryOutboundTracks = /* @__PURE__ */ new Set();
|
|
3578
|
+
dryInboundTracks = /* @__PURE__ */ new Set();
|
|
2988
3579
|
close() {
|
|
2989
|
-
|
|
2990
|
-
this.
|
|
2991
|
-
this.
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
3580
|
+
this.call.activeIssuesRegistry.removeIssueTracker(this);
|
|
3581
|
+
this._lastRaisedAt.clear();
|
|
3582
|
+
this.clear();
|
|
3583
|
+
}
|
|
3584
|
+
add(issue) {
|
|
3585
|
+
if (issue.type === this._config.dryOutboundIssueType) {
|
|
3586
|
+
if (issue.trackId) {
|
|
3587
|
+
this.dryOutboundTracks.add(issue.trackId);
|
|
3588
|
+
}
|
|
3589
|
+
} else if (issue.type === this._config.dryInboundIssueType) {
|
|
3590
|
+
if (issue.trackId) {
|
|
3591
|
+
this.dryInboundTracks.add(issue.trackId);
|
|
3592
|
+
}
|
|
2999
3593
|
}
|
|
3000
|
-
if (this.startedAt === void 0) this.startedAt = minSampleTimestamps;
|
|
3001
|
-
if (this.endedAt === void 0) this.endedAt = maxSampleTimestamps;
|
|
3002
|
-
this.closedAt = Date.now();
|
|
3003
|
-
this.emit("close");
|
|
3004
|
-
this._notify("call-closed", { ...this.eventScope });
|
|
3005
3594
|
}
|
|
3006
|
-
|
|
3007
|
-
if (
|
|
3008
|
-
|
|
3595
|
+
delete(issue) {
|
|
3596
|
+
if (issue.type === this._config.dryOutboundIssueType) {
|
|
3597
|
+
this.dryOutboundTracks.delete(issue.trackId ?? "");
|
|
3598
|
+
} else if (issue.type === this._config.dryInboundIssueType) {
|
|
3599
|
+
this.dryInboundTracks.delete(issue.trackId ?? "");
|
|
3600
|
+
}
|
|
3601
|
+
return true;
|
|
3009
3602
|
}
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3603
|
+
get size() {
|
|
3604
|
+
return this.dryOutboundTracks.size + this.dryInboundTracks.size;
|
|
3605
|
+
}
|
|
3606
|
+
clear() {
|
|
3607
|
+
this.dryOutboundTracks.clear();
|
|
3608
|
+
this.dryInboundTracks.clear();
|
|
3609
|
+
}
|
|
3610
|
+
has(issue) {
|
|
3611
|
+
if (issue.type === this._config.dryOutboundIssueType) {
|
|
3612
|
+
return this.dryOutboundTracks.has(issue.trackId ?? "");
|
|
3613
|
+
} else if (issue.type === this._config.dryInboundIssueType) {
|
|
3614
|
+
return this.dryInboundTracks.has(issue.trackId ?? "");
|
|
3014
3615
|
}
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3616
|
+
return false;
|
|
3617
|
+
}
|
|
3618
|
+
update() {
|
|
3619
|
+
const now = Date.now();
|
|
3620
|
+
const deliveries = /* @__PURE__ */ new Map();
|
|
3621
|
+
for (const client of this.call.observedClients.values()) {
|
|
3622
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
3623
|
+
for (const outboundTrack of peerConnection.observedOutboundTracks.values()) {
|
|
3624
|
+
deliveries.set(outboundTrack.id, {
|
|
3625
|
+
publisherClientId: client.clientId,
|
|
3626
|
+
publisherSending: (outboundTrack.bitrate ?? 0) > 0,
|
|
3627
|
+
publisherDryIssue: this.dryOutboundTracks.has(outboundTrack.id),
|
|
3628
|
+
publisherBitrate: outboundTrack.bitrate ?? 0,
|
|
3629
|
+
numberOfDrySubscribers: 0,
|
|
3630
|
+
numberOfSubscribers: 0
|
|
3631
|
+
});
|
|
3632
|
+
}
|
|
3633
|
+
for (const inboundTrack of peerConnection.observedInboundTracks.values()) {
|
|
3634
|
+
const delivery = deliveries.get(inboundTrack.remoteOutboundTrack?.id ?? "");
|
|
3635
|
+
if (!delivery) {
|
|
3636
|
+
continue;
|
|
3637
|
+
}
|
|
3638
|
+
delivery.numberOfDrySubscribers += this.dryInboundTracks.has(inboundTrack.id) ? 1 : 0;
|
|
3639
|
+
delivery.numberOfSubscribers += 1;
|
|
3640
|
+
}
|
|
3641
|
+
}
|
|
3018
3642
|
}
|
|
3019
|
-
|
|
3020
|
-
|
|
3643
|
+
for (const [outboundTrackId, delivery] of deliveries) {
|
|
3644
|
+
if (delivery.numberOfSubscribers < this._config.minReceivers) {
|
|
3645
|
+
continue;
|
|
3646
|
+
}
|
|
3647
|
+
if (delivery.publisherSending && delivery.numberOfDrySubscribers === 0) continue;
|
|
3648
|
+
const dryRatio = delivery.numberOfDrySubscribers / delivery.numberOfSubscribers;
|
|
3649
|
+
let type;
|
|
3650
|
+
if (!delivery.publisherSending) {
|
|
3651
|
+
type = TrackDeliveryMismatchTypes.publisherTrackDry;
|
|
3652
|
+
} else if (this._config.allReceiversRatio <= dryRatio) {
|
|
3653
|
+
type = TrackDeliveryMismatchTypes.publishedTrackNotDelivered;
|
|
3654
|
+
} else {
|
|
3655
|
+
type = TrackDeliveryMismatchTypes.receiverTrackNotDelivered;
|
|
3656
|
+
}
|
|
3657
|
+
const key = `${outboundTrackId}:${type}`;
|
|
3658
|
+
if (now - (this._lastRaisedAt.get(key) ?? 0) < this._config.cooldownMs) continue;
|
|
3659
|
+
this._lastRaisedAt.set(key, now);
|
|
3660
|
+
this.call.addIssue({
|
|
3661
|
+
type,
|
|
3662
|
+
timestamp: now,
|
|
3663
|
+
payload: {
|
|
3664
|
+
type,
|
|
3665
|
+
trackId: outboundTrackId,
|
|
3666
|
+
publisherClientId: delivery.publisherClientId,
|
|
3667
|
+
publisherSending: delivery.publisherSending,
|
|
3668
|
+
publisherDryIssue: delivery.publisherDryIssue,
|
|
3669
|
+
publisherBitrate: delivery.publisherBitrate,
|
|
3670
|
+
numberOfSubscribers: delivery.numberOfSubscribers,
|
|
3671
|
+
numberOfDrySubscribers: delivery.numberOfDrySubscribers
|
|
3672
|
+
}
|
|
3673
|
+
});
|
|
3021
3674
|
}
|
|
3022
|
-
|
|
3023
|
-
|
|
3675
|
+
}
|
|
3676
|
+
};
|
|
3677
|
+
|
|
3678
|
+
// src/detectors/IssueConclusion.ts
|
|
3679
|
+
var families = [
|
|
3680
|
+
{
|
|
3681
|
+
match: (t) => t.startsWith("congestion") || t.includes("bandwidth"),
|
|
3682
|
+
family: {
|
|
3683
|
+
label: "congestion",
|
|
3684
|
+
infrastructure: {
|
|
3685
|
+
summary: "network congestion is open across independent calls at the same time",
|
|
3686
|
+
recommendation: "check SFU egress bandwidth and host network saturation before looking at any single participant"
|
|
3687
|
+
},
|
|
3688
|
+
call: {
|
|
3689
|
+
summary: "most of this call is congested at once",
|
|
3690
|
+
recommendation: "check the SFU worker serving this call, and whether one publisher is saturating it"
|
|
3691
|
+
}
|
|
3024
3692
|
}
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3693
|
+
},
|
|
3694
|
+
{
|
|
3695
|
+
match: (t) => t.startsWith("ice-") || t.includes("turn") || t === "unstable-ice-path",
|
|
3696
|
+
family: {
|
|
3697
|
+
label: "connectivity",
|
|
3698
|
+
infrastructure: {
|
|
3699
|
+
summary: "ICE connectivity is failing across independent calls at the same time",
|
|
3700
|
+
recommendation: "check SFU reachability and TURN health; a worker restart shows up as a wave of DTLS-stage failures rather than pure consent loss"
|
|
3701
|
+
},
|
|
3702
|
+
call: {
|
|
3703
|
+
summary: "this call is losing ICE connectivity broadly",
|
|
3704
|
+
recommendation: "check the SFU worker and the transports belonging to this call"
|
|
3705
|
+
}
|
|
3706
|
+
}
|
|
3707
|
+
},
|
|
3708
|
+
{
|
|
3709
|
+
match: (t) => t.startsWith("audio-") || t.includes("concealment") || t.includes("jitter-buffer"),
|
|
3710
|
+
family: {
|
|
3711
|
+
label: "audio",
|
|
3712
|
+
infrastructure: {
|
|
3713
|
+
summary: "audio impairment is open across independent calls at the same time",
|
|
3714
|
+
recommendation: "check SFU egress and the audio forwarding path; concealment rising fleet-wide is a delivery symptom, not a device one"
|
|
3715
|
+
},
|
|
3716
|
+
call: {
|
|
3717
|
+
summary: "audio is impaired for most of this call",
|
|
3718
|
+
recommendation: "check this call's SFU worker; if only receivers are affected, the publisher's uplink is the next place to look"
|
|
3719
|
+
},
|
|
3720
|
+
publishedTrack: {
|
|
3721
|
+
summary: "the subscribers of one published audio track are impaired while others are fine",
|
|
3722
|
+
recommendation: "check that publisher's uplink and the SFU's forwarding of that producer; the receivers themselves are exonerated by the fan-out"
|
|
3723
|
+
}
|
|
3724
|
+
}
|
|
3725
|
+
},
|
|
3726
|
+
{
|
|
3727
|
+
match: (t) => t.includes("video") || t.includes("freeze") || t.includes("keyframe") || t.includes("decoder"),
|
|
3728
|
+
family: {
|
|
3729
|
+
label: "video delivery",
|
|
3730
|
+
infrastructure: {
|
|
3731
|
+
summary: "video delivery is degraded across independent calls at the same time",
|
|
3732
|
+
recommendation: "check SFU egress and the keyframe/PLI path; a keyframe storm fleet-wide usually means the server is dropping or not answering PLIs"
|
|
3733
|
+
},
|
|
3734
|
+
call: {
|
|
3735
|
+
summary: "video is degraded for most of this call",
|
|
3736
|
+
recommendation: "check this call's SFU worker and whether PLIs are being answered"
|
|
3737
|
+
},
|
|
3738
|
+
publishedTrack: {
|
|
3739
|
+
summary: "the subscribers of one published video track are degraded while others are fine",
|
|
3740
|
+
recommendation: "check that publisher's uplink and the forwarding of that producer; consider recreating the consumers if only some receivers are affected"
|
|
3741
|
+
}
|
|
3742
|
+
}
|
|
3743
|
+
},
|
|
3744
|
+
{
|
|
3745
|
+
match: (t) => t.includes("cpu") || t.includes("overloaded") || t.includes("encoder") || t.includes("performance"),
|
|
3746
|
+
family: {
|
|
3747
|
+
label: "endpoint capacity",
|
|
3748
|
+
// Breadth here does NOT implicate the servers. Endpoint CPU is owned by the endpoint, so the
|
|
3749
|
+
// same symptom in many unrelated calls points at what those endpoints have in common —
|
|
3750
|
+
// a client release, a browser update, a VDI host — not at the SFU.
|
|
3751
|
+
crossCallDomain: "client-population",
|
|
3752
|
+
infrastructure: {
|
|
3753
|
+
summary: "endpoint CPU limitation is open across independent calls at the same time",
|
|
3754
|
+
recommendation: "this is not an SFU symptom: look at what the affected clients share \u2014 a recent client release, a browser version, or shared/virtualised hardware"
|
|
3755
|
+
},
|
|
3756
|
+
call: {
|
|
3757
|
+
summary: "most participants of this call are CPU limited",
|
|
3758
|
+
recommendation: "check whether the call is asking too much of endpoints (layer count, resolution, participant grid size)"
|
|
3759
|
+
}
|
|
3760
|
+
}
|
|
3761
|
+
}
|
|
3762
|
+
];
|
|
3763
|
+
var unknownFamily = { label: "issue" };
|
|
3764
|
+
function familyOf(issueType) {
|
|
3765
|
+
return families.find((entry) => entry.match(issueType))?.family ?? unknownFamily;
|
|
3766
|
+
}
|
|
3767
|
+
function callConfidenceOf(spread) {
|
|
3768
|
+
let confidence = 0.3;
|
|
3769
|
+
if (0 < spread.totalClients) {
|
|
3770
|
+
confidence += Math.min(0.3, spread.affectedClients / spread.totalClients * 0.3);
|
|
3771
|
+
}
|
|
3772
|
+
if (spread.onsetBurst) confidence += 0.2;
|
|
3773
|
+
if (spread.publishedTrackId !== void 0) confidence += 0.1;
|
|
3774
|
+
return Math.min(1, Math.round(confidence * 100) / 100);
|
|
3775
|
+
}
|
|
3776
|
+
function observerConfidenceOf(spread) {
|
|
3777
|
+
let confidence = 0.3 + Math.min(0.4, Math.max(0, spread.affectedCalls - 1) * 0.1);
|
|
3778
|
+
if (spread.onsetBurst) confidence += 0.2;
|
|
3779
|
+
return Math.min(1, Math.round(confidence * 100) / 100);
|
|
3780
|
+
}
|
|
3781
|
+
function concludeCallIssue(spread) {
|
|
3782
|
+
const family = familyOf(spread.issueType);
|
|
3783
|
+
const confidence = callConfidenceOf(spread);
|
|
3784
|
+
const affectedShare = 0 < spread.totalClients ? ` (${spread.affectedClients}/${spread.totalClients} clients)` : "";
|
|
3785
|
+
if (spread.publishedTrackId !== void 0) {
|
|
3786
|
+
const text = family.publishedTrack;
|
|
3787
|
+
return {
|
|
3788
|
+
faultDomain: "published-track",
|
|
3789
|
+
confidence,
|
|
3790
|
+
summary: text ? `${text.summary} \u2014 track ${spread.publishedTrackId}${affectedShare}` : `'${spread.issueType}' follows published track ${spread.publishedTrackId}${affectedShare}, so the fault is on that source's path rather than the receivers'`,
|
|
3791
|
+
recommendation: text?.recommendation
|
|
3792
|
+
};
|
|
3793
|
+
}
|
|
3794
|
+
if (1 < spread.affectedClients) {
|
|
3795
|
+
const text = family.call;
|
|
3796
|
+
return {
|
|
3797
|
+
faultDomain: "call",
|
|
3798
|
+
confidence,
|
|
3799
|
+
summary: text ? `${text.summary}${affectedShare}` : `'${spread.issueType}' is open for ${spread.affectedClients} of ${spread.totalClients} participants of one call`,
|
|
3800
|
+
recommendation: text?.recommendation
|
|
3801
|
+
};
|
|
3802
|
+
}
|
|
3803
|
+
return {
|
|
3804
|
+
faultDomain: spread.affectedClients === 1 ? "endpoint" : "unknown",
|
|
3805
|
+
confidence,
|
|
3806
|
+
summary: `'${spread.issueType}' affects a single endpoint; nothing is shared with the other participants`,
|
|
3807
|
+
recommendation: "treat as that participant's own device or last mile"
|
|
3808
|
+
};
|
|
3809
|
+
}
|
|
3810
|
+
function concludeObserverIssue(spread) {
|
|
3811
|
+
const family = familyOf(spread.issueType);
|
|
3812
|
+
const confidence = observerConfidenceOf(spread);
|
|
3813
|
+
const affectedShare = 0 < spread.totalClients ? ` (${spread.affectedClients}/${spread.totalClients} clients)` : "";
|
|
3814
|
+
if (spread.affectedCalls <= 1) {
|
|
3815
|
+
return {
|
|
3816
|
+
faultDomain: "call",
|
|
3817
|
+
confidence,
|
|
3818
|
+
summary: `'${spread.issueType}' is confined to a single call${affectedShare}; that is a meeting problem, not a fleet one`,
|
|
3819
|
+
recommendation: "look at the call-scoped finding for this call rather than at the infrastructure"
|
|
3820
|
+
};
|
|
3821
|
+
}
|
|
3822
|
+
const domain = family.crossCallDomain ?? "infrastructure";
|
|
3823
|
+
const text = family.infrastructure;
|
|
3824
|
+
return {
|
|
3825
|
+
faultDomain: domain,
|
|
3826
|
+
confidence,
|
|
3827
|
+
summary: text ? `${text.summary} \u2014 ${spread.affectedCalls} of ${spread.totalCalls} calls${affectedShare}` : `'${spread.issueType}' is open in ${spread.affectedCalls} of ${spread.totalCalls} independent calls at once${affectedShare}; they share only the infrastructure`,
|
|
3828
|
+
recommendation: text?.recommendation
|
|
3829
|
+
};
|
|
3830
|
+
}
|
|
3831
|
+
|
|
3832
|
+
// src/detectors/CallConcurrentIssueDetector.ts
|
|
3833
|
+
var CallConcurrentIssueTypes = {
|
|
3834
|
+
/** Several participants of this call have the same issue open **at the same time**. */
|
|
3835
|
+
concurrentClientIssues: "CONCURRENT_CLIENT_ISSUES",
|
|
3836
|
+
/** Those issues also *began* together — the signature of one shared event. */
|
|
3837
|
+
issueOnsetBurst: "ISSUE_ONSET_BURST"
|
|
3838
|
+
};
|
|
3839
|
+
var CallConcurrentIssueDetector = class _CallConcurrentIssueDetector {
|
|
3840
|
+
constructor(_call, config = {}) {
|
|
3841
|
+
this._call = _call;
|
|
3842
|
+
this._config = {
|
|
3843
|
+
issueTypes: [],
|
|
3844
|
+
minClients: 3,
|
|
3845
|
+
minAffectedClients: 3,
|
|
3846
|
+
affectedRatioThreshold: 0.5,
|
|
3847
|
+
onsetBurstWindowInMs: 2e3,
|
|
3848
|
+
cooldownMs: 6e4,
|
|
3849
|
+
...config
|
|
3850
|
+
};
|
|
3851
|
+
for (const type of this._config.issueTypes) {
|
|
3852
|
+
this._call.activeIssuesRegistry.addIssueTracker(type, this);
|
|
3853
|
+
}
|
|
3854
|
+
}
|
|
3855
|
+
_call;
|
|
3856
|
+
static NAME = "call-concurrent-issue-detector";
|
|
3857
|
+
name = _CallConcurrentIssueDetector.NAME;
|
|
3858
|
+
_config;
|
|
3859
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
3860
|
+
/** issue type -> the issues of that type currently open in this call. */
|
|
3861
|
+
_byType = /* @__PURE__ */ new Map();
|
|
3862
|
+
_size = 0;
|
|
3863
|
+
/** The groups that qualified on the most recent `update()`. Exposed for tests/dashboards. */
|
|
3864
|
+
lastGroups = [];
|
|
3865
|
+
get size() {
|
|
3866
|
+
return this._size;
|
|
3867
|
+
}
|
|
3868
|
+
has(issue) {
|
|
3869
|
+
return this._byType.get(issue.type)?.has(issue) ?? false;
|
|
3870
|
+
}
|
|
3871
|
+
add(issue) {
|
|
3872
|
+
let bucket = this._byType.get(issue.type);
|
|
3873
|
+
if (!bucket) {
|
|
3874
|
+
bucket = /* @__PURE__ */ new Set();
|
|
3875
|
+
this._byType.set(issue.type, bucket);
|
|
3876
|
+
}
|
|
3877
|
+
if (bucket.has(issue)) return;
|
|
3878
|
+
bucket.add(issue);
|
|
3879
|
+
++this._size;
|
|
3880
|
+
}
|
|
3881
|
+
delete(issue) {
|
|
3882
|
+
const bucket = this._byType.get(issue.type);
|
|
3883
|
+
if (!bucket?.delete(issue)) return false;
|
|
3884
|
+
--this._size;
|
|
3885
|
+
if (bucket.size === 0) this._byType.delete(issue.type);
|
|
3886
|
+
return true;
|
|
3887
|
+
}
|
|
3888
|
+
clear() {
|
|
3889
|
+
this._byType.clear();
|
|
3890
|
+
this._size = 0;
|
|
3891
|
+
this.lastGroups = [];
|
|
3892
|
+
}
|
|
3893
|
+
update() {
|
|
3894
|
+
this.lastGroups = [];
|
|
3895
|
+
if (this._size === 0) return;
|
|
3896
|
+
const now = Date.now();
|
|
3897
|
+
const totalClients = this._call.observedClients.size;
|
|
3898
|
+
if (totalClients < this._config.minClients) return;
|
|
3899
|
+
for (const [type, issues] of this._byType) {
|
|
3900
|
+
const group = this._groupOf(type, issues, totalClients);
|
|
3901
|
+
if (group.clientIds.length < this._config.minAffectedClients) continue;
|
|
3902
|
+
if (group.affectedRatio < this._config.affectedRatioThreshold) continue;
|
|
3903
|
+
this.lastGroups.push(group);
|
|
3904
|
+
if (now - (this._lastRaisedAt.get(type) ?? 0) < this._config.cooldownMs) continue;
|
|
3905
|
+
this._lastRaisedAt.set(type, now);
|
|
3906
|
+
const burst = group.onsetSpreadInMs <= this._config.onsetBurstWindowInMs;
|
|
3907
|
+
const issueType = burst ? CallConcurrentIssueTypes.issueOnsetBurst : CallConcurrentIssueTypes.concurrentClientIssues;
|
|
3908
|
+
const conclusion = concludeCallIssue({
|
|
3909
|
+
issueType: type,
|
|
3910
|
+
affectedClients: group.clientIds.length,
|
|
3911
|
+
totalClients: group.totalClients,
|
|
3912
|
+
onsetBurst: burst
|
|
3913
|
+
});
|
|
3914
|
+
this._call.addIssue({
|
|
3915
|
+
type: issueType,
|
|
3916
|
+
timestamp: now,
|
|
3917
|
+
payload: {
|
|
3918
|
+
type: issueType,
|
|
3919
|
+
issueType: type,
|
|
3920
|
+
scope: "call",
|
|
3921
|
+
conclusion,
|
|
3922
|
+
callId: this._call.callId,
|
|
3923
|
+
clients: group.totalClients,
|
|
3924
|
+
affectedClients: group.clientIds.length,
|
|
3925
|
+
affectedRatio: group.affectedRatio,
|
|
3926
|
+
affectedClientIds: group.clientIds,
|
|
3927
|
+
onsetSpreadInMs: group.onsetSpreadInMs,
|
|
3928
|
+
onsetBurst: burst,
|
|
3929
|
+
firstObservedAt: group.firstObservedAt
|
|
3930
|
+
}
|
|
3931
|
+
});
|
|
3932
|
+
}
|
|
3933
|
+
}
|
|
3934
|
+
close() {
|
|
3935
|
+
this._call.activeIssuesRegistry.removeIssueTracker(this);
|
|
3936
|
+
this._lastRaisedAt.clear();
|
|
3937
|
+
this.clear();
|
|
3938
|
+
}
|
|
3939
|
+
_groupOf(type, issues, totalClients) {
|
|
3940
|
+
const clientIds = /* @__PURE__ */ new Set();
|
|
3941
|
+
const list = [];
|
|
3942
|
+
let earliest = Infinity;
|
|
3943
|
+
let latest = -Infinity;
|
|
3944
|
+
for (const issue of issues) {
|
|
3945
|
+
list.push(issue);
|
|
3946
|
+
clientIds.add(issue.clientId);
|
|
3947
|
+
if (issue.observedAt < earliest) earliest = issue.observedAt;
|
|
3948
|
+
if (latest < issue.observedAt) latest = issue.observedAt;
|
|
3949
|
+
}
|
|
3950
|
+
return {
|
|
3951
|
+
type,
|
|
3952
|
+
issues: list,
|
|
3953
|
+
clientIds: [...clientIds],
|
|
3954
|
+
totalClients,
|
|
3955
|
+
affectedRatio: 0 < totalClients ? clientIds.size / totalClients : 0,
|
|
3956
|
+
onsetSpreadInMs: latest - earliest,
|
|
3957
|
+
firstObservedAt: earliest
|
|
3958
|
+
};
|
|
3959
|
+
}
|
|
3960
|
+
};
|
|
3961
|
+
|
|
3962
|
+
// src/detectors/IssueFanOutDetector.ts
|
|
3963
|
+
var IssueFanOutTypes = {
|
|
3964
|
+
/** Most receivers of one published track have the same issue open → the fault follows the source. */
|
|
3965
|
+
publishedTrackIssueFanOut: "PUBLISHED_TRACK_ISSUE_FAN_OUT",
|
|
3966
|
+
/** Exactly one receiver of a track has it → that receiver's own problem. */
|
|
3967
|
+
singleReceiverIssue: "SINGLE_RECEIVER_ISSUE"
|
|
3968
|
+
};
|
|
3969
|
+
var IssueFanOutDetector = class _IssueFanOutDetector {
|
|
3970
|
+
constructor(_call, config = {}) {
|
|
3971
|
+
this._call = _call;
|
|
3972
|
+
this._config = {
|
|
3973
|
+
issueTypes: [],
|
|
3974
|
+
minReceivers: 3,
|
|
3975
|
+
affectedRatioThreshold: 0.6,
|
|
3976
|
+
reportSingleReceiver: true,
|
|
3977
|
+
cooldownMs: 6e4,
|
|
3978
|
+
...config
|
|
3979
|
+
};
|
|
3980
|
+
for (const type of this._config.issueTypes) {
|
|
3981
|
+
this._call.activeIssuesRegistry.addIssueTracker(type, this);
|
|
3982
|
+
}
|
|
3983
|
+
}
|
|
3984
|
+
_call;
|
|
3985
|
+
static NAME = "issue-fan-out-detector";
|
|
3986
|
+
name = _IssueFanOutDetector.NAME;
|
|
3987
|
+
_config;
|
|
3988
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
3989
|
+
/** Open issues that name a track. Issues without a `trackId` cannot be attributed and are dropped. */
|
|
3990
|
+
_trackIssues = /* @__PURE__ */ new Set();
|
|
3991
|
+
get size() {
|
|
3992
|
+
return this._trackIssues.size;
|
|
3993
|
+
}
|
|
3994
|
+
has(issue) {
|
|
3995
|
+
return this._trackIssues.has(issue);
|
|
3996
|
+
}
|
|
3997
|
+
add(issue) {
|
|
3998
|
+
if (issue.trackId === void 0) return;
|
|
3999
|
+
this._trackIssues.add(issue);
|
|
4000
|
+
}
|
|
4001
|
+
delete(issue) {
|
|
4002
|
+
return this._trackIssues.delete(issue);
|
|
4003
|
+
}
|
|
4004
|
+
clear() {
|
|
4005
|
+
this._trackIssues.clear();
|
|
4006
|
+
}
|
|
4007
|
+
update() {
|
|
4008
|
+
if (this._trackIssues.size === 0) return;
|
|
4009
|
+
if (!this._call.remoteTrackResolver) return;
|
|
4010
|
+
const now = Date.now();
|
|
4011
|
+
const byPublisher = /* @__PURE__ */ new Map();
|
|
4012
|
+
for (const issue of this._trackIssues) {
|
|
4013
|
+
const publisher = this._publisherOf(issue);
|
|
4014
|
+
if (!publisher) continue;
|
|
4015
|
+
let byType = byPublisher.get(publisher);
|
|
4016
|
+
if (!byType) {
|
|
4017
|
+
byType = /* @__PURE__ */ new Map();
|
|
4018
|
+
byPublisher.set(publisher, byType);
|
|
4019
|
+
}
|
|
4020
|
+
const clientIds = byType.get(issue.type) ?? /* @__PURE__ */ new Set();
|
|
4021
|
+
clientIds.add(issue.clientId);
|
|
4022
|
+
byType.set(issue.type, clientIds);
|
|
4023
|
+
}
|
|
4024
|
+
for (const [publisher, byType] of byPublisher) {
|
|
4025
|
+
const numberOfReceivers = publisher.remoteInboundTracks.size;
|
|
4026
|
+
if (numberOfReceivers === 0) continue;
|
|
4027
|
+
for (const [issueType, clientIds] of byType) {
|
|
4028
|
+
const affectedRatio = clientIds.size / numberOfReceivers;
|
|
4029
|
+
const isFanOut = this._config.minReceivers <= numberOfReceivers && this._config.affectedRatioThreshold <= affectedRatio;
|
|
4030
|
+
const isSingle = this._config.reportSingleReceiver && clientIds.size === 1 && 1 < numberOfReceivers;
|
|
4031
|
+
if (!isFanOut && !isSingle) continue;
|
|
4032
|
+
const key = `${publisher.id}:${issueType}`;
|
|
4033
|
+
if (now - (this._lastRaisedAt.get(key) ?? 0) < this._config.cooldownMs) continue;
|
|
4034
|
+
this._lastRaisedAt.set(key, now);
|
|
4035
|
+
const type = isFanOut ? IssueFanOutTypes.publishedTrackIssueFanOut : IssueFanOutTypes.singleReceiverIssue;
|
|
4036
|
+
const conclusion = concludeCallIssue({
|
|
4037
|
+
issueType,
|
|
4038
|
+
affectedClients: clientIds.size,
|
|
4039
|
+
totalClients: numberOfReceivers,
|
|
4040
|
+
onsetBurst: false,
|
|
4041
|
+
publishedTrackId: isFanOut ? publisher.id : void 0
|
|
4042
|
+
});
|
|
4043
|
+
this._call.addIssue({
|
|
4044
|
+
type,
|
|
4045
|
+
timestamp: now,
|
|
4046
|
+
payload: {
|
|
4047
|
+
type,
|
|
4048
|
+
issueType,
|
|
4049
|
+
conclusion,
|
|
4050
|
+
trackId: publisher.id,
|
|
4051
|
+
kind: publisher.kind,
|
|
4052
|
+
publisherClientId: publisher.getPeerConnection().client.clientId,
|
|
4053
|
+
// The corroborating half: if the source's own egress looks fine while its
|
|
4054
|
+
// receivers do not, the SFU/forwarding path is implicated rather than the sender.
|
|
4055
|
+
publisherBitrate: publisher.bitrate,
|
|
4056
|
+
publisherDegraded: publisher.degraded,
|
|
4057
|
+
publisherDegradedReasons: publisher.degradedReasons,
|
|
4058
|
+
receivers: numberOfReceivers,
|
|
4059
|
+
affectedReceivers: clientIds.size,
|
|
4060
|
+
affectedRatio,
|
|
4061
|
+
affectedClientIds: [...clientIds]
|
|
4062
|
+
}
|
|
4063
|
+
});
|
|
4064
|
+
}
|
|
4065
|
+
}
|
|
4066
|
+
}
|
|
4067
|
+
close() {
|
|
4068
|
+
this._call.activeIssuesRegistry.removeIssueTracker(this);
|
|
4069
|
+
this._lastRaisedAt.clear();
|
|
4070
|
+
this.clear();
|
|
4071
|
+
}
|
|
4072
|
+
/**
|
|
4073
|
+
* Resolve the issue's `trackId` to the outbound track that published it.
|
|
4074
|
+
*
|
|
4075
|
+
* Looked up through the reporting client's own peer connections rather than by scanning the call:
|
|
4076
|
+
* the issue names its client, so the search is bounded by that client's transports (typically one
|
|
4077
|
+
* or two) instead of by the size of the meeting.
|
|
4078
|
+
*/
|
|
4079
|
+
_publisherOf(issue) {
|
|
4080
|
+
const client = this._call.observedClients.get(issue.clientId);
|
|
4081
|
+
if (!client || issue.trackId === void 0) return void 0;
|
|
4082
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
4083
|
+
const inboundTrack = peerConnection.observedInboundTracks.get(issue.trackId);
|
|
4084
|
+
if (inboundTrack) return inboundTrack.remoteOutboundTrack;
|
|
4085
|
+
}
|
|
4086
|
+
return void 0;
|
|
4087
|
+
}
|
|
4088
|
+
};
|
|
4089
|
+
|
|
4090
|
+
// src/detectors/PublisherFaultCorroborationDetector.ts
|
|
4091
|
+
var PublisherFaultTypes = {
|
|
4092
|
+
/**
|
|
4093
|
+
* A publisher is reporting trouble on its own send path **while** its subscribers report trouble
|
|
4094
|
+
* receiving it. Both ends agree, so the source is implicated rather than inferred.
|
|
4095
|
+
*/
|
|
4096
|
+
corroboratedPublisherFault: "CORROBORATED_PUBLISHER_FAULT"
|
|
4097
|
+
};
|
|
4098
|
+
var PublisherFaultCorroborationDetector = class _PublisherFaultCorroborationDetector {
|
|
4099
|
+
constructor(_call, config = {}) {
|
|
4100
|
+
this._call = _call;
|
|
4101
|
+
this._config = {
|
|
4102
|
+
publisherIssueTypes: [],
|
|
4103
|
+
receiverIssueTypes: [],
|
|
4104
|
+
minAffectedReceivers: 2,
|
|
4105
|
+
cooldownMs: 6e4,
|
|
4106
|
+
...config
|
|
4107
|
+
};
|
|
4108
|
+
for (const type of this._config.publisherIssueTypes) {
|
|
4109
|
+
this._call.activeIssuesRegistry.addIssueTracker(type, this);
|
|
4110
|
+
}
|
|
4111
|
+
for (const type of this._config.receiverIssueTypes) {
|
|
4112
|
+
this._call.activeIssuesRegistry.addIssueTracker(type, this);
|
|
4113
|
+
}
|
|
4114
|
+
}
|
|
4115
|
+
_call;
|
|
4116
|
+
static NAME = "publisher-fault-corroboration-detector";
|
|
4117
|
+
name = _PublisherFaultCorroborationDetector.NAME;
|
|
4118
|
+
_config;
|
|
4119
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
4120
|
+
/** Open publisher-side issues that name a track. */
|
|
4121
|
+
_publisherIssues = /* @__PURE__ */ new Set();
|
|
4122
|
+
/** Open receiver-side issues that name a track. */
|
|
4123
|
+
_receiverIssues = /* @__PURE__ */ new Set();
|
|
4124
|
+
/** The faults corroborated on the most recent `update()`. Exposed for tests/dashboards. */
|
|
4125
|
+
lastFaults = [];
|
|
4126
|
+
get size() {
|
|
4127
|
+
return this._publisherIssues.size + this._receiverIssues.size;
|
|
4128
|
+
}
|
|
4129
|
+
has(issue) {
|
|
4130
|
+
return this._publisherIssues.has(issue) || this._receiverIssues.has(issue);
|
|
4131
|
+
}
|
|
4132
|
+
add(issue) {
|
|
4133
|
+
if (issue.trackId === void 0) return;
|
|
4134
|
+
if (this._config.publisherIssueTypes.includes(issue.type)) this._publisherIssues.add(issue);
|
|
4135
|
+
else if (this._config.receiverIssueTypes.includes(issue.type)) this._receiverIssues.add(issue);
|
|
4136
|
+
}
|
|
4137
|
+
delete(issue) {
|
|
4138
|
+
return this._publisherIssues.delete(issue) || this._receiverIssues.delete(issue);
|
|
4139
|
+
}
|
|
4140
|
+
clear() {
|
|
4141
|
+
this._publisherIssues.clear();
|
|
4142
|
+
this._receiverIssues.clear();
|
|
4143
|
+
this.lastFaults = [];
|
|
4144
|
+
}
|
|
4145
|
+
update() {
|
|
4146
|
+
this.lastFaults = [];
|
|
4147
|
+
if (this._publisherIssues.size === 0 || this._receiverIssues.size === 0) return;
|
|
4148
|
+
if (!this._call.remoteTrackResolver) return;
|
|
4149
|
+
const now = Date.now();
|
|
4150
|
+
const suspects = /* @__PURE__ */ new Map();
|
|
4151
|
+
for (const issue of this._publisherIssues) {
|
|
4152
|
+
const publisher = this._outboundTrackOf(issue);
|
|
4153
|
+
if (!publisher) continue;
|
|
4154
|
+
const types = suspects.get(publisher) ?? /* @__PURE__ */ new Set();
|
|
4155
|
+
types.add(issue.type);
|
|
4156
|
+
suspects.set(publisher, types);
|
|
4157
|
+
}
|
|
4158
|
+
if (suspects.size === 0) return;
|
|
4159
|
+
const receiverIssuesByTrackId = /* @__PURE__ */ new Map();
|
|
4160
|
+
for (const issue of this._receiverIssues) {
|
|
4161
|
+
const existing = receiverIssuesByTrackId.get(issue.trackId);
|
|
4162
|
+
if (existing) existing.push(issue);
|
|
4163
|
+
else receiverIssuesByTrackId.set(issue.trackId, [issue]);
|
|
4164
|
+
}
|
|
4165
|
+
for (const [publisher, publisherIssueTypes] of suspects) {
|
|
4166
|
+
const receivers = publisher.remoteInboundTracks;
|
|
4167
|
+
if (receivers.size === 0) continue;
|
|
4168
|
+
const affectedClientIds = /* @__PURE__ */ new Set();
|
|
4169
|
+
const receiverIssueTypes = /* @__PURE__ */ new Set();
|
|
4170
|
+
for (const receiver of receivers) {
|
|
4171
|
+
const issues = receiverIssuesByTrackId.get(receiver.id);
|
|
4172
|
+
if (!issues) continue;
|
|
4173
|
+
for (const issue of issues) {
|
|
4174
|
+
affectedClientIds.add(issue.clientId);
|
|
4175
|
+
receiverIssueTypes.add(issue.type);
|
|
4176
|
+
}
|
|
4177
|
+
}
|
|
4178
|
+
if (affectedClientIds.size < this._config.minAffectedReceivers) continue;
|
|
4179
|
+
const fault = {
|
|
4180
|
+
trackId: publisher.id,
|
|
4181
|
+
kind: publisher.kind,
|
|
4182
|
+
publisherClientId: publisher.getPeerConnection().client.clientId,
|
|
4183
|
+
publisherIssueTypes: [...publisherIssueTypes],
|
|
4184
|
+
receiverIssueTypes: [...receiverIssueTypes],
|
|
4185
|
+
receivers: receivers.size,
|
|
4186
|
+
affectedReceivers: affectedClientIds.size,
|
|
4187
|
+
affectedClientIds: [...affectedClientIds],
|
|
4188
|
+
publisherBitrate: publisher.bitrate
|
|
4189
|
+
};
|
|
4190
|
+
this.lastFaults.push(fault);
|
|
4191
|
+
if (now - (this._lastRaisedAt.get(publisher.id) ?? 0) < this._config.cooldownMs) continue;
|
|
4192
|
+
this._lastRaisedAt.set(publisher.id, now);
|
|
4193
|
+
this._call.addIssue({
|
|
4194
|
+
type: PublisherFaultTypes.corroboratedPublisherFault,
|
|
4195
|
+
timestamp: now,
|
|
4196
|
+
payload: {
|
|
4197
|
+
type: PublisherFaultTypes.corroboratedPublisherFault,
|
|
4198
|
+
callId: this._call.callId,
|
|
4199
|
+
...fault,
|
|
4200
|
+
conclusion: {
|
|
4201
|
+
faultDomain: "published-track",
|
|
4202
|
+
summary: `${fault.publisherClientId} reports ${fault.publisherIssueTypes.join(", ")} on track ${fault.trackId} while ${fault.affectedReceivers} of ${fault.receivers} subscribers report ${fault.receiverIssueTypes.join(", ")} \u2014 both ends agree`,
|
|
4203
|
+
recommendation: "the source is implicated, not inferred: check that publisher's capture, encoder and uplink before looking at the SFU or the receivers",
|
|
4204
|
+
// Higher than any single-ended finding: two independent parties, one conclusion.
|
|
4205
|
+
confidence: 0.9
|
|
4206
|
+
}
|
|
4207
|
+
}
|
|
4208
|
+
});
|
|
4209
|
+
}
|
|
4210
|
+
}
|
|
4211
|
+
close() {
|
|
4212
|
+
this._call.activeIssuesRegistry.removeIssueTracker(this);
|
|
4213
|
+
this._lastRaisedAt.clear();
|
|
4214
|
+
this.clear();
|
|
4215
|
+
}
|
|
4216
|
+
/**
|
|
4217
|
+
* Resolve a publisher-side issue's `trackId` to the outbound track it is about.
|
|
4218
|
+
*
|
|
4219
|
+
* Looked up through the reporting client's own peer connections: the issue names its client, so
|
|
4220
|
+
* the search is bounded by that client's transports rather than by the size of the call.
|
|
4221
|
+
*/
|
|
4222
|
+
_outboundTrackOf(issue) {
|
|
4223
|
+
const client = this._call.observedClients.get(issue.clientId);
|
|
4224
|
+
if (!client || issue.trackId === void 0) return void 0;
|
|
4225
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
4226
|
+
const outboundTrack = peerConnection.observedOutboundTracks.get(issue.trackId);
|
|
4227
|
+
if (outboundTrack) return outboundTrack;
|
|
4228
|
+
}
|
|
4229
|
+
return void 0;
|
|
4230
|
+
}
|
|
4231
|
+
};
|
|
4232
|
+
|
|
4233
|
+
// src/ObservedCall.ts
|
|
4234
|
+
var logger4 = createLogger("ObservedCall");
|
|
4235
|
+
var ObservedCall = class extends import_events3.EventEmitter {
|
|
4236
|
+
constructor(settings, observer, activeIssuesRegistry) {
|
|
4237
|
+
super();
|
|
4238
|
+
this.observer = observer;
|
|
4239
|
+
this.activeIssuesRegistry = activeIssuesRegistry;
|
|
4240
|
+
this.setMaxListeners(Infinity);
|
|
4241
|
+
this.eventScope = { observer: this.observer, observedCall: this };
|
|
4242
|
+
this.callId = settings.callId;
|
|
4243
|
+
this.appData = settings.appData ?? {};
|
|
4244
|
+
this.scoreCalculator = new DefaultCallScoreCalculator(this);
|
|
4245
|
+
this.detectors = new Detectors();
|
|
4246
|
+
this.settings = {
|
|
4247
|
+
closeCallIfEmptyForMs: settings.closeCallIfEmptyForMs,
|
|
4248
|
+
autoUpdateOnClientUpdate: settings.autoUpdateOnClientUpdate ?? true
|
|
4249
|
+
};
|
|
4250
|
+
}
|
|
4251
|
+
observer;
|
|
4252
|
+
activeIssuesRegistry;
|
|
4253
|
+
scoreCalculator;
|
|
4254
|
+
detectors;
|
|
4255
|
+
callId;
|
|
4256
|
+
observedClients = /* @__PURE__ */ new Map();
|
|
4257
|
+
clientsUsedTurn = /* @__PURE__ */ new Set();
|
|
4258
|
+
calculatedScore = {
|
|
4259
|
+
weight: 1,
|
|
4260
|
+
value: void 0
|
|
4261
|
+
};
|
|
4262
|
+
remoteTrackResolver;
|
|
4263
|
+
/**
|
|
4264
|
+
* Published tracks that currently have **no** subscriber linked to them.
|
|
4265
|
+
*
|
|
4266
|
+
* Maintained by the `RemoteTrackResolver` at the exact moments a track gains or loses its last
|
|
4267
|
+
* subscriber — the only moments the answer can change. `UnconsumedTrackDetector` reads this
|
|
4268
|
+
* instead of walking every published track in the call, so in a healthy call (where the set is
|
|
4269
|
+
* empty) it does no work at all.
|
|
4270
|
+
*
|
|
4271
|
+
* Empty when no resolver is configured: without links, "no subscribers" is unknowable.
|
|
4272
|
+
*/
|
|
4273
|
+
unconsumedOutboundTracks = /* @__PURE__ */ new Set();
|
|
4274
|
+
totalAddedClients = 0;
|
|
4275
|
+
totalRemovedClients = 0;
|
|
4276
|
+
numberOfIssues = 0;
|
|
4277
|
+
numberOfPeerConnections = 0;
|
|
4278
|
+
numberOfInboundRtpStreams = 0;
|
|
4279
|
+
numberOfOutboundRtpStreams = 0;
|
|
4280
|
+
numberOfDataChannels = 0;
|
|
4281
|
+
maxNumberOfClients = 0;
|
|
4282
|
+
deltaNumberOfIssues = 0;
|
|
4283
|
+
// public deltaRttLt50Measurements = 0;
|
|
4284
|
+
// public deltaRttLt150Measurements = 0;
|
|
4285
|
+
// public deltaRttLt300Measurements = 0;
|
|
4286
|
+
// public deltaRttGtOrEq300Measurements = 0;
|
|
4287
|
+
appData;
|
|
4288
|
+
closed = false;
|
|
4289
|
+
startedAt;
|
|
4290
|
+
endedAt;
|
|
4291
|
+
closedAt;
|
|
4292
|
+
settings;
|
|
4293
|
+
/** Ancestry base shared by all Observer-bus events originating at this call. */
|
|
4294
|
+
eventScope;
|
|
4295
|
+
closeTimer;
|
|
4296
|
+
get numberOfClients() {
|
|
4297
|
+
return this.observedClients.size;
|
|
4298
|
+
}
|
|
4299
|
+
get score() {
|
|
4300
|
+
return this.calculatedScore.value;
|
|
4301
|
+
}
|
|
4302
|
+
addDetector(name, config = {}) {
|
|
4303
|
+
if (this.closed) return this;
|
|
4304
|
+
let detector;
|
|
4305
|
+
switch (name) {
|
|
4306
|
+
case UnconsumedTrackDetector.NAME: {
|
|
4307
|
+
detector = new UnconsumedTrackDetector(this, config);
|
|
4308
|
+
break;
|
|
4309
|
+
}
|
|
4310
|
+
case TrackDeliveryMismatchDetector.NAME: {
|
|
4311
|
+
detector = new TrackDeliveryMismatchDetector(this, config);
|
|
4312
|
+
break;
|
|
4313
|
+
}
|
|
4314
|
+
case CallConcurrentIssueDetector.NAME: {
|
|
4315
|
+
detector = new CallConcurrentIssueDetector(this, config);
|
|
4316
|
+
break;
|
|
4317
|
+
}
|
|
4318
|
+
case IssueFanOutDetector.NAME: {
|
|
4319
|
+
detector = new IssueFanOutDetector(this, config);
|
|
4320
|
+
break;
|
|
4321
|
+
}
|
|
4322
|
+
case PublisherFaultCorroborationDetector.NAME: {
|
|
4323
|
+
detector = new PublisherFaultCorroborationDetector(this, config);
|
|
4324
|
+
break;
|
|
4325
|
+
}
|
|
4326
|
+
default: {
|
|
4327
|
+
logger4.warn("Unknown detector name %s; skipping", name);
|
|
4328
|
+
return this;
|
|
4329
|
+
}
|
|
4330
|
+
}
|
|
4331
|
+
this.detectors.add(detector);
|
|
4332
|
+
return this;
|
|
4333
|
+
}
|
|
4334
|
+
/**
|
|
4335
|
+
* Raise a call-level (server-side) finding; surfaced on the Observer bus as `call-issue`.
|
|
4336
|
+
*
|
|
4337
|
+
* `payload` takes an **object** — it is delivered to in-process handlers, so there is nothing to
|
|
4338
|
+
* serialise for. Pass a string only if you already have one.
|
|
4339
|
+
*/
|
|
4340
|
+
addIssue(issue) {
|
|
4341
|
+
if (this.closed) return;
|
|
4342
|
+
this._notify("call-issue", { ...this.eventScope, issue });
|
|
4343
|
+
}
|
|
4344
|
+
close() {
|
|
4345
|
+
if (this.closed) return;
|
|
4346
|
+
this.update();
|
|
4347
|
+
this.closed = true;
|
|
4348
|
+
let minSampleTimestamps;
|
|
4349
|
+
let maxSampleTimestamps;
|
|
4350
|
+
const clients = [...this.observedClients.values()];
|
|
4351
|
+
for (const client of clients) {
|
|
4352
|
+
client.close();
|
|
4353
|
+
if (client.joinedAt) minSampleTimestamps = Math.min(minSampleTimestamps ?? client.joinedAt, client.joinedAt);
|
|
4354
|
+
if (client.leftAt) maxSampleTimestamps = Math.max(maxSampleTimestamps ?? client.leftAt, client.leftAt);
|
|
4355
|
+
}
|
|
4356
|
+
if (this.startedAt === void 0) this.startedAt = minSampleTimestamps;
|
|
4357
|
+
if (this.endedAt === void 0) this.endedAt = maxSampleTimestamps;
|
|
4358
|
+
this.closedAt = Date.now();
|
|
4359
|
+
this.detectors.clear();
|
|
4360
|
+
this.activeIssuesRegistry.clear();
|
|
4361
|
+
this.emit("close");
|
|
4362
|
+
this._notify("call-closed", { ...this.eventScope });
|
|
4363
|
+
}
|
|
4364
|
+
getObservedClient(clientId) {
|
|
4365
|
+
if (this.closed || !this.observedClients.has(clientId)) return;
|
|
4366
|
+
return this.observedClients.get(clientId);
|
|
4367
|
+
}
|
|
4368
|
+
createObservedClient(settings) {
|
|
4369
|
+
if (this.closed) {
|
|
4370
|
+
logger4.warn("Attempted to create a client (clientId: %s) on a closed call %s", settings.clientId, this.callId);
|
|
4371
|
+
return void 0;
|
|
4372
|
+
}
|
|
4373
|
+
if (this.observedClients.has(settings.clientId)) {
|
|
4374
|
+
logger4.warn("Client with id %s already exists in call %s; returning the existing instance", settings.clientId, this.callId);
|
|
4375
|
+
return this.observedClients.get(settings.clientId);
|
|
4376
|
+
}
|
|
4377
|
+
if (!settings.closeClientIfIdleForMs) {
|
|
4378
|
+
settings.closeClientIfIdleForMs = this.observer.config.closeClientIfIdleForMs;
|
|
4379
|
+
}
|
|
4380
|
+
if (settings.appData === void 0) {
|
|
4381
|
+
settings.appData = this.observer.config.createClientAppData?.({ clientId: settings.clientId, observedCall: this });
|
|
4382
|
+
}
|
|
4383
|
+
const observedClientIssueRegistry = new ObservedClientIssueRegistry(this.activeIssuesRegistry);
|
|
4384
|
+
const result = new ObservedClient(
|
|
4385
|
+
settings,
|
|
4386
|
+
this,
|
|
4387
|
+
observedClientIssueRegistry
|
|
4388
|
+
);
|
|
4389
|
+
const wasEmpty = this.observedClients.size === 0;
|
|
4390
|
+
const onUpdate = (...args) => this._onClientUpdate(result, args[2]);
|
|
4391
|
+
const joined = () => this._clientJoined(result);
|
|
3029
4392
|
const left = () => this._clientLeft(result);
|
|
3030
4393
|
result.once("close", () => {
|
|
3031
4394
|
result.off("update", onUpdate);
|
|
3032
4395
|
result.off("joined", joined);
|
|
3033
4396
|
result.off("left", left);
|
|
3034
4397
|
this.observedClients.delete(settings.clientId);
|
|
4398
|
+
this.clientsUsedTurn.delete(settings.clientId);
|
|
3035
4399
|
if (this.observedClients.size === 0) {
|
|
3036
4400
|
this.emit("empty");
|
|
3037
4401
|
this._notify("call-empty", { ...this.eventScope });
|
|
@@ -3085,11 +4449,13 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
3085
4449
|
this._notify("call-updated", { ...this.eventScope, context });
|
|
3086
4450
|
this.deltaNumberOfIssues = 0;
|
|
3087
4451
|
}
|
|
3088
|
-
_onClientUpdate(client) {
|
|
4452
|
+
_onClientUpdate(client, context) {
|
|
3089
4453
|
this.deltaNumberOfIssues += client.deltaNumberOfIssues;
|
|
3090
4454
|
this.numberOfIssues += client.deltaNumberOfIssues;
|
|
3091
|
-
if (client.usingTURN)
|
|
3092
|
-
|
|
4455
|
+
if (client.usingTURN) this.clientsUsedTurn.add(client.clientId);
|
|
4456
|
+
else this.clientsUsedTurn.delete(client.clientId);
|
|
4457
|
+
if (this.settings.autoUpdateOnClientUpdate) {
|
|
4458
|
+
this.update(context);
|
|
3093
4459
|
}
|
|
3094
4460
|
}
|
|
3095
4461
|
_clientJoined(client) {
|
|
@@ -3198,21 +4564,22 @@ var ObservedTURN = class extends import_events4.EventEmitter {
|
|
|
3198
4564
|
this.numberOfClients = clientIds.size;
|
|
3199
4565
|
}
|
|
3200
4566
|
addPeerConnection(peerConnection) {
|
|
3201
|
-
const turnPairs = peerConnection.
|
|
4567
|
+
const turnPairs = peerConnection.selectedIceCandiadtePairForTurn;
|
|
3202
4568
|
if (turnPairs.length !== 1) {
|
|
3203
4569
|
return logger5.warn(`Expected exactly one TURN pair, but found for peerconnection ${peerConnection.peerConnectionId}`, turnPairs.length), void 0;
|
|
3204
4570
|
}
|
|
3205
4571
|
const candidatePair = turnPairs[0];
|
|
3206
|
-
const rawUrl = candidatePair.
|
|
4572
|
+
const rawUrl = candidatePair.getLocalCandidate()?.url;
|
|
3207
4573
|
if (!rawUrl) {
|
|
3208
|
-
return logger5.warn(`No
|
|
4574
|
+
return logger5.warn(`No local candidate URL found for peerconnection ${peerConnection.peerConnectionId}`), void 0;
|
|
3209
4575
|
}
|
|
3210
|
-
const
|
|
3211
|
-
const turnServerUrl = `${turnUrl.protocol}//${turnUrl.hostname}:${turnUrl.port}`;
|
|
4576
|
+
const turnServerUrl = rawUrl.split("?")[0];
|
|
3212
4577
|
let turnServer = this.servers.get(turnServerUrl);
|
|
3213
4578
|
if (!turnServer) {
|
|
3214
4579
|
turnServer = new ObservedTurnServer(turnServerUrl, this);
|
|
4580
|
+
this.servers.set(turnServerUrl, turnServer);
|
|
3215
4581
|
}
|
|
4582
|
+
turnServer.observedPeerConnections.set(peerConnection.peerConnectionId, peerConnection);
|
|
3216
4583
|
return turnServer;
|
|
3217
4584
|
}
|
|
3218
4585
|
removePeerConnection(peerConnection) {
|
|
@@ -3222,80 +4589,6 @@ var ObservedTURN = class extends import_events4.EventEmitter {
|
|
|
3222
4589
|
}
|
|
3223
4590
|
};
|
|
3224
4591
|
|
|
3225
|
-
// src/updaters/OnAllCallObserverUpdater.ts
|
|
3226
|
-
var OnAllCallObserverUpdater = class {
|
|
3227
|
-
constructor(observer) {
|
|
3228
|
-
this.observer = observer;
|
|
3229
|
-
this._onNewObservedCall = this._onNewObservedCall.bind(this);
|
|
3230
|
-
this.observer.once("observer-closed", () => {
|
|
3231
|
-
this.observer.off("call-added", this._onNewObservedCall);
|
|
3232
|
-
});
|
|
3233
|
-
this.observer.on("call-added", this._onNewObservedCall);
|
|
3234
|
-
}
|
|
3235
|
-
observer;
|
|
3236
|
-
name = "OnAllCallObserverUpdater";
|
|
3237
|
-
description = "Call Observer's update() method when all of the ObservedCalls are updated";
|
|
3238
|
-
_updatedCalls = /* @__PURE__ */ new Set();
|
|
3239
|
-
closed = false;
|
|
3240
|
-
close() {
|
|
3241
|
-
if (this.closed) return;
|
|
3242
|
-
this.closed = true;
|
|
3243
|
-
this._updatedCalls.clear();
|
|
3244
|
-
}
|
|
3245
|
-
_onNewObservedCall({ observedCall }) {
|
|
3246
|
-
if (this.closed) return;
|
|
3247
|
-
const onUpdate = () => this._onObservedCallUpdated(observedCall);
|
|
3248
|
-
observedCall.once("close", () => {
|
|
3249
|
-
observedCall.off("update", onUpdate);
|
|
3250
|
-
this._updatedCalls.delete(observedCall.callId);
|
|
3251
|
-
this._updateIfEveryCallUpdated();
|
|
3252
|
-
});
|
|
3253
|
-
observedCall.on("update", onUpdate);
|
|
3254
|
-
}
|
|
3255
|
-
_onObservedCallUpdated(observedCall) {
|
|
3256
|
-
if (observedCall.closed) return;
|
|
3257
|
-
this._updatedCalls.add(observedCall.callId);
|
|
3258
|
-
this._updateIfEveryCallUpdated();
|
|
3259
|
-
}
|
|
3260
|
-
_updateIfEveryCallUpdated() {
|
|
3261
|
-
if (this._updatedCalls.size < this.observer.observedCalls.size) return;
|
|
3262
|
-
else if (this.closed) return;
|
|
3263
|
-
this._updatedCalls.clear();
|
|
3264
|
-
this.observer.update();
|
|
3265
|
-
}
|
|
3266
|
-
};
|
|
3267
|
-
|
|
3268
|
-
// src/updaters/OnAnyCallObserverUpdater.ts
|
|
3269
|
-
var OnAnyCallObserverUpdater = class {
|
|
3270
|
-
constructor(observver) {
|
|
3271
|
-
this.observver = observver;
|
|
3272
|
-
this._onNewObservedCall = this._onNewObservedCall.bind(this);
|
|
3273
|
-
this.observver.once("observer-closed", () => {
|
|
3274
|
-
this.observver.off("call-added", this._onNewObservedCall);
|
|
3275
|
-
});
|
|
3276
|
-
this.observver.on("call-added", this._onNewObservedCall);
|
|
3277
|
-
}
|
|
3278
|
-
observver;
|
|
3279
|
-
name = "OnAnyCallObserverUpdater";
|
|
3280
|
-
description = "Call Observer's update() method on any of the ObservedCall is updated";
|
|
3281
|
-
closed = false;
|
|
3282
|
-
close() {
|
|
3283
|
-
if (this.closed) return;
|
|
3284
|
-
this.closed = true;
|
|
3285
|
-
}
|
|
3286
|
-
_onNewObservedCall({ observedCall }) {
|
|
3287
|
-
if (this.closed) return;
|
|
3288
|
-
const onUpdate = () => {
|
|
3289
|
-
if (this.closed) return;
|
|
3290
|
-
this.observver.update();
|
|
3291
|
-
};
|
|
3292
|
-
observedCall.once("close", () => {
|
|
3293
|
-
observedCall.off("update", onUpdate);
|
|
3294
|
-
});
|
|
3295
|
-
observedCall.on("update", onUpdate);
|
|
3296
|
-
}
|
|
3297
|
-
};
|
|
3298
|
-
|
|
3299
4592
|
// src/common/Middleware.ts
|
|
3300
4593
|
var Executor = class {
|
|
3301
4594
|
constructor(stack, finalCallback) {
|
|
@@ -3362,10 +4655,19 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3362
4655
|
sample;
|
|
3363
4656
|
webrtcTransportIds = /* @__PURE__ */ new Set();
|
|
3364
4657
|
closed = false;
|
|
4658
|
+
// Id -> sample indexes over the arrays in `this.sample`, so an entity can be reached in O(1)
|
|
4659
|
+
// instead of scanning (`sample.producers.find(...)`). They hold the *same* objects as the arrays.
|
|
4660
|
+
_transportSamples = /* @__PURE__ */ new Map();
|
|
4661
|
+
_producerSamples = /* @__PURE__ */ new Map();
|
|
4662
|
+
_consumerSamples = /* @__PURE__ */ new Map();
|
|
4663
|
+
_dataProducerSamples = /* @__PURE__ */ new Map();
|
|
4664
|
+
_dataConsumerSamples = /* @__PURE__ */ new Map();
|
|
4665
|
+
_enrich;
|
|
3365
4666
|
constructor(settings) {
|
|
3366
4667
|
super();
|
|
3367
4668
|
this.router = settings.router;
|
|
3368
4669
|
this.appData = settings.appData ?? {};
|
|
4670
|
+
this._enrich = settings.enrich;
|
|
3369
4671
|
this.sample = {
|
|
3370
4672
|
routerId: this.router.id,
|
|
3371
4673
|
attachments: settings.attachments ?? {},
|
|
@@ -3384,6 +4686,52 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3384
4686
|
get attachments() {
|
|
3385
4687
|
return this.sample.attachments;
|
|
3386
4688
|
}
|
|
4689
|
+
/* ---------------------------------------------------------------------------------------------
|
|
4690
|
+
* Reaching individual entity samples.
|
|
4691
|
+
*
|
|
4692
|
+
* `sample.producers` (and friends) are arrays because that is the shape you want to serialize.
|
|
4693
|
+
* These accessors index the same objects by id, so annotating one entity is an O(1) lookup and a
|
|
4694
|
+
* mutation, rather than a scan that silently does nothing when the id is wrong.
|
|
4695
|
+
* ------------------------------------------------------------------------------------------- */
|
|
4696
|
+
getTransportSample(id) {
|
|
4697
|
+
return this._transportSamples.get(id);
|
|
4698
|
+
}
|
|
4699
|
+
getProducerSample(id) {
|
|
4700
|
+
return this._producerSamples.get(id);
|
|
4701
|
+
}
|
|
4702
|
+
getConsumerSample(id) {
|
|
4703
|
+
return this._consumerSamples.get(id);
|
|
4704
|
+
}
|
|
4705
|
+
getDataProducerSample(id) {
|
|
4706
|
+
return this._dataProducerSamples.get(id);
|
|
4707
|
+
}
|
|
4708
|
+
getDataConsumerSample(id) {
|
|
4709
|
+
return this._dataConsumerSamples.get(id);
|
|
4710
|
+
}
|
|
4711
|
+
/**
|
|
4712
|
+
* Merge `attachments` into an entity's sample, whichever kind it is.
|
|
4713
|
+
*
|
|
4714
|
+
* Ids are unique across mediasoup entity kinds, so one method covers all of them. Returns `false`
|
|
4715
|
+
* when the id is unknown — a real answer instead of failing quietly, which matters when the
|
|
4716
|
+
* annotation is driven by application events that may race the mediasoup ones.
|
|
4717
|
+
*/
|
|
4718
|
+
attachTo(id, attachments) {
|
|
4719
|
+
const sample = this._transportSamples.get(id) ?? this._producerSamples.get(id) ?? this._consumerSamples.get(id) ?? this._dataProducerSamples.get(id) ?? this._dataConsumerSamples.get(id);
|
|
4720
|
+
if (!sample) return false;
|
|
4721
|
+
sample.attachments = { ...sample.attachments, ...attachments };
|
|
4722
|
+
return true;
|
|
4723
|
+
}
|
|
4724
|
+
/**
|
|
4725
|
+
* A **detached deep copy** of the current sample — the basis for building your own report.
|
|
4726
|
+
*
|
|
4727
|
+
* `this.sample` is live: its arrays grow and its `history` entries are appended as the router
|
|
4728
|
+
* runs, so a report built directly on it keeps changing after you think you're done. This returns
|
|
4729
|
+
* a snapshot that never moves.
|
|
4730
|
+
*/
|
|
4731
|
+
snapshot() {
|
|
4732
|
+
if (typeof structuredClone === "function") return structuredClone(this.sample);
|
|
4733
|
+
return JSON.parse(JSON.stringify(this.sample));
|
|
4734
|
+
}
|
|
3387
4735
|
close() {
|
|
3388
4736
|
if (this.closed) return;
|
|
3389
4737
|
this.closed = true;
|
|
@@ -3421,7 +4769,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3421
4769
|
tuple: transport.iceSelectedTuple,
|
|
3422
4770
|
history
|
|
3423
4771
|
};
|
|
3424
|
-
this.
|
|
4772
|
+
this._addTransportSample(transportSample, transport);
|
|
3425
4773
|
this.webrtcTransportIds.add(transport.id);
|
|
3426
4774
|
const onIceStateChange = (iceState) => history.push({ type: `icestate-changed-to-${iceState}`, timestamp: Date.now() });
|
|
3427
4775
|
const onIceSelectedTupleChange = (tuple) => {
|
|
@@ -3455,7 +4803,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3455
4803
|
rtcpTuple: transport.rtcpTuple,
|
|
3456
4804
|
history
|
|
3457
4805
|
};
|
|
3458
|
-
this.
|
|
4806
|
+
this._addTransportSample(transportSample, transport);
|
|
3459
4807
|
const onTuple = (tuple) => {
|
|
3460
4808
|
transportSample.tuple = tuple;
|
|
3461
4809
|
if (transportSample.connectedAt === void 0) transportSample.connectedAt = Date.now();
|
|
@@ -3487,7 +4835,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3487
4835
|
tuple: transport.tuple,
|
|
3488
4836
|
history
|
|
3489
4837
|
};
|
|
3490
|
-
this.
|
|
4838
|
+
this._addTransportSample(transportSample, transport);
|
|
3491
4839
|
const onSctpStateChange = (sctpState) => {
|
|
3492
4840
|
if (sctpState === "connected" && transportSample.connectedAt === void 0) transportSample.connectedAt = Date.now();
|
|
3493
4841
|
history.push({ type: `sctpstate-changed-to-${sctpState}`, timestamp: Date.now() });
|
|
@@ -3506,7 +4854,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3506
4854
|
connectedAt: now,
|
|
3507
4855
|
history: []
|
|
3508
4856
|
};
|
|
3509
|
-
this.
|
|
4857
|
+
this._addTransportSample(transportSample, transport);
|
|
3510
4858
|
this._attachTransportObserverListeners(transport, transportSample);
|
|
3511
4859
|
}
|
|
3512
4860
|
addProducer(transport, producer) {
|
|
@@ -3528,13 +4876,14 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3528
4876
|
rids: producer.rtpParameters.encodings?.map((e) => e.rid).filter((r) => typeof r === "string"),
|
|
3529
4877
|
history: []
|
|
3530
4878
|
};
|
|
3531
|
-
this.
|
|
4879
|
+
this._addProducerSample(producerSample, producer, transport);
|
|
3532
4880
|
const onPause = () => producerSample.history.push({ type: "pause", timestamp: Date.now() });
|
|
3533
4881
|
const onResume = () => producerSample.history.push({ type: "resume", timestamp: Date.now() });
|
|
3534
4882
|
producer.observer.once("close", () => {
|
|
3535
4883
|
producer.observer.off("pause", onPause);
|
|
3536
4884
|
producer.observer.off("resume", onResume);
|
|
3537
4885
|
producerSample.closedAt = Date.now();
|
|
4886
|
+
this.emit("producer-sample-closed", { sample: producerSample, producer, transport });
|
|
3538
4887
|
});
|
|
3539
4888
|
producer.observer.on("pause", onPause);
|
|
3540
4889
|
producer.observer.on("resume", onResume);
|
|
@@ -3548,7 +4897,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3548
4897
|
kind: consumer.kind,
|
|
3549
4898
|
history: []
|
|
3550
4899
|
};
|
|
3551
|
-
this.
|
|
4900
|
+
this._addConsumerSample(consumerSample, consumer, transport);
|
|
3552
4901
|
const onPause = () => consumerSample.history.push({ type: "pause", timestamp: Date.now() });
|
|
3553
4902
|
const onResume = () => consumerSample.history.push({ type: "resume", timestamp: Date.now() });
|
|
3554
4903
|
const onProducerPause = () => consumerSample.history.push({ type: "producerPaused", timestamp: Date.now() });
|
|
@@ -3559,6 +4908,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3559
4908
|
consumer.off("producerresume", onProducerResume);
|
|
3560
4909
|
consumer.off("producerpause", onProducerPause);
|
|
3561
4910
|
consumerSample.closedAt = Date.now();
|
|
4911
|
+
this.emit("consumer-sample-closed", { sample: consumerSample, consumer, transport });
|
|
3562
4912
|
});
|
|
3563
4913
|
consumer.observer.on("pause", onPause);
|
|
3564
4914
|
consumer.observer.on("resume", onResume);
|
|
@@ -3573,8 +4923,11 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3573
4923
|
label: dataProducer.label,
|
|
3574
4924
|
protocol: dataProducer.protocol
|
|
3575
4925
|
};
|
|
3576
|
-
this.
|
|
3577
|
-
dataProducer.observer.once("close", () =>
|
|
4926
|
+
this._addDataProducerSample(dataProducerSample, dataProducer, transport);
|
|
4927
|
+
dataProducer.observer.once("close", () => {
|
|
4928
|
+
dataProducerSample.closedAt = Date.now();
|
|
4929
|
+
this.emit("data-producer-sample-closed", { sample: dataProducerSample, dataProducer, transport });
|
|
4930
|
+
});
|
|
3578
4931
|
}
|
|
3579
4932
|
addDataConsumer(transport, dataConsumer) {
|
|
3580
4933
|
const dataConsumerSample = {
|
|
@@ -3585,8 +4938,11 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3585
4938
|
label: dataConsumer.label,
|
|
3586
4939
|
protocol: dataConsumer.protocol
|
|
3587
4940
|
};
|
|
3588
|
-
this.
|
|
3589
|
-
dataConsumer.observer.once("close", () =>
|
|
4941
|
+
this._addDataConsumerSample(dataConsumerSample, dataConsumer, transport);
|
|
4942
|
+
dataConsumer.observer.once("close", () => {
|
|
4943
|
+
dataConsumerSample.closedAt = Date.now();
|
|
4944
|
+
this.emit("data-consumer-sample-closed", { sample: dataConsumerSample, dataConsumer, transport });
|
|
4945
|
+
});
|
|
3590
4946
|
}
|
|
3591
4947
|
attachRouterListeners() {
|
|
3592
4948
|
this.router.observer.once("close", () => {
|
|
@@ -3596,6 +4952,56 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3596
4952
|
});
|
|
3597
4953
|
this.router.observer.on("newtransport", this.addTransport);
|
|
3598
4954
|
}
|
|
4955
|
+
/* ---------------------------------------------------------------------------------------------
|
|
4956
|
+
* Registration: push into the sample array, index by id, run the enricher, announce.
|
|
4957
|
+
* The order matters — enrichment lands *before* the event, so a listener always sees the
|
|
4958
|
+
* declaratively-attached data and can build on it rather than race it.
|
|
4959
|
+
* ------------------------------------------------------------------------------------------- */
|
|
4960
|
+
_addTransportSample(sample, transport) {
|
|
4961
|
+
this.sample.transports.push(sample);
|
|
4962
|
+
this._transportSamples.set(sample.id, sample);
|
|
4963
|
+
this._applyEnrichment(sample, this._enrich?.transport && (() => this._enrich?.transport?.(transport)));
|
|
4964
|
+
this.emit("transport-sample-added", { sample, transport });
|
|
4965
|
+
}
|
|
4966
|
+
_addProducerSample(sample, producer, transport) {
|
|
4967
|
+
this.sample.producers.push(sample);
|
|
4968
|
+
this._producerSamples.set(sample.id, sample);
|
|
4969
|
+
this._applyEnrichment(sample, this._enrich?.producer && (() => this._enrich?.producer?.(producer, transport)));
|
|
4970
|
+
this.emit("producer-sample-added", { sample, producer, transport });
|
|
4971
|
+
}
|
|
4972
|
+
_addConsumerSample(sample, consumer, transport) {
|
|
4973
|
+
this.sample.consumers.push(sample);
|
|
4974
|
+
this._consumerSamples.set(sample.id, sample);
|
|
4975
|
+
this._applyEnrichment(sample, this._enrich?.consumer && (() => this._enrich?.consumer?.(consumer, transport)));
|
|
4976
|
+
this.emit("consumer-sample-added", { sample, consumer, transport });
|
|
4977
|
+
}
|
|
4978
|
+
_addDataProducerSample(sample, dataProducer, transport) {
|
|
4979
|
+
this.sample.dataProducers.push(sample);
|
|
4980
|
+
this._dataProducerSamples.set(sample.id, sample);
|
|
4981
|
+
this._applyEnrichment(sample, this._enrich?.dataProducer && (() => this._enrich?.dataProducer?.(dataProducer, transport)));
|
|
4982
|
+
this.emit("data-producer-sample-added", { sample, dataProducer, transport });
|
|
4983
|
+
}
|
|
4984
|
+
_addDataConsumerSample(sample, dataConsumer, transport) {
|
|
4985
|
+
this.sample.dataConsumers.push(sample);
|
|
4986
|
+
this._dataConsumerSamples.set(sample.id, sample);
|
|
4987
|
+
this._applyEnrichment(sample, this._enrich?.dataConsumer && (() => this._enrich?.dataConsumer?.(dataConsumer, transport)));
|
|
4988
|
+
this.emit("data-consumer-sample-added", { sample, dataConsumer, transport });
|
|
4989
|
+
}
|
|
4990
|
+
/**
|
|
4991
|
+
* Run an enricher and merge what it returns.
|
|
4992
|
+
*
|
|
4993
|
+
* Takes a thunk rather than a value so the **invocation** is inside the guard — application code
|
|
4994
|
+
* runs here, and a throwing enricher must not take the router's bookkeeping down with it.
|
|
4995
|
+
*/
|
|
4996
|
+
_applyEnrichment(sample, enrich) {
|
|
4997
|
+
if (!enrich) return;
|
|
4998
|
+
try {
|
|
4999
|
+
const attachments = enrich();
|
|
5000
|
+
if (attachments) sample.attachments = { ...sample.attachments, ...attachments };
|
|
5001
|
+
} catch (err) {
|
|
5002
|
+
logger6.warn("Mediasoup sample enricher threw for router %s: %o", this.id, err);
|
|
5003
|
+
}
|
|
5004
|
+
}
|
|
3599
5005
|
_attachTransportObserverListeners(transport, transportSample, onClose) {
|
|
3600
5006
|
const onNewProducer = (producer) => this.addProducer(transport, producer);
|
|
3601
5007
|
const onNewConsumer = (consumer) => this.addConsumer(transport, consumer);
|
|
@@ -3608,6 +5014,7 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3608
5014
|
transport.observer.off("newdataconsumer", onNewDataConsumer);
|
|
3609
5015
|
onClose?.();
|
|
3610
5016
|
transportSample.closedAt = Date.now();
|
|
5017
|
+
this.emit("transport-sample-closed", { sample: transportSample, transport });
|
|
3611
5018
|
});
|
|
3612
5019
|
transport.observer.on("newproducer", onNewProducer);
|
|
3613
5020
|
transport.observer.on("newconsumer", onNewConsumer);
|
|
@@ -3616,36 +5023,1282 @@ var ObservedMediasoupRouter = class extends import_events5.EventEmitter {
|
|
|
3616
5023
|
}
|
|
3617
5024
|
};
|
|
3618
5025
|
|
|
3619
|
-
// src/
|
|
3620
|
-
var logger7 = createLogger("
|
|
3621
|
-
var
|
|
3622
|
-
constructor(
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
5026
|
+
// src/issues/ActiveIssuesRegistry.ts
|
|
5027
|
+
var logger7 = createLogger("ActiveIssuesRegistry");
|
|
5028
|
+
var ActiveIssuesRegistry = class {
|
|
5029
|
+
constructor(parent) {
|
|
5030
|
+
this.parent = parent;
|
|
5031
|
+
}
|
|
5032
|
+
parent;
|
|
5033
|
+
issues = /* @__PURE__ */ new Set();
|
|
5034
|
+
typesToTrackers = /* @__PURE__ */ new Map();
|
|
5035
|
+
get size() {
|
|
5036
|
+
return this.issues.size;
|
|
5037
|
+
}
|
|
5038
|
+
/**
|
|
5039
|
+
* The open issues in this scope, in insertion order.
|
|
5040
|
+
*
|
|
5041
|
+
* Insertion order is age order (`observedAt` is assigned on insert), which is what lets a consumer
|
|
5042
|
+
* stop at the first entry newer than its cutoff instead of scanning the whole set.
|
|
5043
|
+
*/
|
|
5044
|
+
values() {
|
|
5045
|
+
return this.issues.values();
|
|
5046
|
+
}
|
|
5047
|
+
[Symbol.iterator]() {
|
|
5048
|
+
return this.issues.values();
|
|
5049
|
+
}
|
|
5050
|
+
has(issue) {
|
|
5051
|
+
return this.issues.has(issue);
|
|
5052
|
+
}
|
|
5053
|
+
add(issue) {
|
|
5054
|
+
if (this.issues.has(issue)) return this;
|
|
5055
|
+
this.issues.add(issue);
|
|
5056
|
+
this.parent?.add(issue);
|
|
5057
|
+
this._trackIssue(issue);
|
|
5058
|
+
return this;
|
|
3641
5059
|
}
|
|
3642
|
-
|
|
3643
|
-
|
|
5060
|
+
delete(issue) {
|
|
5061
|
+
if (!this.issues.delete(issue)) return false;
|
|
5062
|
+
this.parent?.delete(issue);
|
|
5063
|
+
this._untrackIssue(issue);
|
|
5064
|
+
return true;
|
|
5065
|
+
}
|
|
5066
|
+
/** Feed `tracker` every issue of `type` as it opens and closes. One call per type; no wildcard. */
|
|
5067
|
+
addIssueTracker(type, tracker) {
|
|
5068
|
+
this.typesToTrackers.set(
|
|
5069
|
+
type,
|
|
5070
|
+
(this.typesToTrackers.get(type) ?? []).concat(tracker)
|
|
5071
|
+
);
|
|
5072
|
+
return this;
|
|
5073
|
+
}
|
|
5074
|
+
removeIssueTracker(tracker) {
|
|
5075
|
+
for (const [type, trackers] of [...this.typesToTrackers]) {
|
|
5076
|
+
const remaining = trackers.filter((candidate) => candidate !== tracker);
|
|
5077
|
+
if (remaining.length === 0) this.typesToTrackers.delete(type);
|
|
5078
|
+
else this.typesToTrackers.set(type, remaining);
|
|
5079
|
+
}
|
|
5080
|
+
return this;
|
|
5081
|
+
}
|
|
5082
|
+
/**
|
|
5083
|
+
* Drop every issue in this scope, e.g. because the call closed.
|
|
5084
|
+
*
|
|
5085
|
+
* Deletes through {@link delete} so the parent sheds exactly this scope's issues. Tracker
|
|
5086
|
+
* *registrations* survive: a detector subscribed to the observer's registry must keep receiving
|
|
5087
|
+
* issues after any one call ends.
|
|
5088
|
+
*/
|
|
5089
|
+
clear() {
|
|
5090
|
+
for (const issue of [...this.issues]) this.delete(issue);
|
|
5091
|
+
}
|
|
5092
|
+
_trackIssue(issue) {
|
|
5093
|
+
this._trackersOf(issue.type, (tracker) => tracker.add(issue));
|
|
5094
|
+
}
|
|
5095
|
+
_untrackIssue(issue) {
|
|
5096
|
+
this._trackersOf(issue.type, (tracker) => tracker.delete(issue));
|
|
5097
|
+
}
|
|
5098
|
+
/**
|
|
5099
|
+
* Apply `apply` to every tracker interested in `type`.
|
|
5100
|
+
*
|
|
5101
|
+
* A tracker throwing must not abort the fan-out: the issue has already been added to (or removed
|
|
5102
|
+
* from) this registry, so a partial dispatch would leave the remaining trackers permanently out of
|
|
5103
|
+
* step with it. One broken detector should not desynchronise the others.
|
|
5104
|
+
*/
|
|
5105
|
+
_trackersOf(type, apply) {
|
|
5106
|
+
const trackers = this.typesToTrackers.get(type);
|
|
5107
|
+
if (trackers) for (const tracker of trackers) this._safely(tracker, apply);
|
|
5108
|
+
}
|
|
5109
|
+
_safely(target, apply) {
|
|
5110
|
+
try {
|
|
5111
|
+
apply(target);
|
|
5112
|
+
} catch (err) {
|
|
5113
|
+
logger7.warn("An issue tracker threw while being notified; continuing with the rest. %o", err);
|
|
5114
|
+
}
|
|
5115
|
+
}
|
|
5116
|
+
};
|
|
5117
|
+
|
|
5118
|
+
// src/detectors/SfuCongestionDetector.ts
|
|
5119
|
+
var SfuCongestionDetector = class _SfuCongestionDetector {
|
|
5120
|
+
constructor(_observer, config = {}) {
|
|
5121
|
+
this._observer = _observer;
|
|
5122
|
+
this._config = {
|
|
5123
|
+
consumedClientIssueTypes: [
|
|
5124
|
+
"congestion"
|
|
5125
|
+
],
|
|
5126
|
+
emittedObserverIssueType: "sfu-congestion",
|
|
5127
|
+
samplesSendingTimeInMs: 1e4,
|
|
5128
|
+
historySize: 30,
|
|
5129
|
+
minHistorySize: 5,
|
|
5130
|
+
minAffectedClients: 3,
|
|
5131
|
+
minAbsoluteRatioIncrease: 0.05,
|
|
5132
|
+
minRelativeRatioIncrease: 2,
|
|
5133
|
+
robustZThreshold: 3,
|
|
5134
|
+
...config
|
|
5135
|
+
};
|
|
5136
|
+
for (const issueType of this._config.consumedClientIssueTypes) {
|
|
5137
|
+
this._observer.activeIssuesRegistry.addIssueTracker(issueType, this);
|
|
5138
|
+
}
|
|
5139
|
+
this.timer = setInterval(() => {
|
|
5140
|
+
this._closeBucket(Date.now());
|
|
5141
|
+
this._evaluateLatestBucket();
|
|
5142
|
+
}, this._config.samplesSendingTimeInMs);
|
|
5143
|
+
this.timer.unref?.();
|
|
5144
|
+
}
|
|
5145
|
+
_observer;
|
|
5146
|
+
static NAME = "sfu-congestion-detector";
|
|
5147
|
+
name = _SfuCongestionDetector.NAME;
|
|
5148
|
+
_config;
|
|
5149
|
+
_history = [];
|
|
5150
|
+
_trackedIssues = /* @__PURE__ */ new Set();
|
|
5151
|
+
timer;
|
|
5152
|
+
_lastEvaluatedBucket;
|
|
5153
|
+
get size() {
|
|
5154
|
+
return this._trackedIssues.size;
|
|
5155
|
+
}
|
|
5156
|
+
/** Record the issue against the bucket currently open. The timer, not this, closes the bucket. */
|
|
5157
|
+
add(issue) {
|
|
5158
|
+
this._trackedIssues.add(issue);
|
|
5159
|
+
}
|
|
5160
|
+
/**
|
|
5161
|
+
* Deliberately a no-op returning `false`.
|
|
5162
|
+
*
|
|
5163
|
+
* Resolutions are not interesting here. A congested client typically fixes its own symptom by
|
|
5164
|
+
* dropping bitrate hard, so the issue closes within seconds — but it still *happened*, and it is
|
|
5165
|
+
* evidence that the server was under pressure during this bucket. What matters is how many
|
|
5166
|
+
* distinct clients reported congestion within the bucket and whether that count suddenly jumps,
|
|
5167
|
+
* not how long any one client's issue stayed open.
|
|
5168
|
+
*
|
|
5169
|
+
* Nothing leaks: the tracked set is emptied wholesale every time a bucket closes.
|
|
5170
|
+
*/
|
|
5171
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
5172
|
+
delete(_issue) {
|
|
5173
|
+
return false;
|
|
5174
|
+
}
|
|
5175
|
+
clear() {
|
|
5176
|
+
this._trackedIssues.clear();
|
|
5177
|
+
this._history.length = 0;
|
|
5178
|
+
this._lastEvaluatedBucket = void 0;
|
|
5179
|
+
}
|
|
5180
|
+
has(issue) {
|
|
5181
|
+
return this._trackedIssues.has(issue);
|
|
5182
|
+
}
|
|
5183
|
+
close() {
|
|
5184
|
+
if (this.timer) {
|
|
5185
|
+
clearInterval(this.timer);
|
|
5186
|
+
this.timer = void 0;
|
|
5187
|
+
}
|
|
5188
|
+
this._observer.activeIssuesRegistry.removeIssueTracker(this);
|
|
5189
|
+
this.clear();
|
|
5190
|
+
}
|
|
5191
|
+
/** The completed buckets kept so far, oldest first. Read-only — for introspection/tests. */
|
|
5192
|
+
get history() {
|
|
5193
|
+
return this._history;
|
|
5194
|
+
}
|
|
5195
|
+
/**
|
|
5196
|
+
* Intentionally empty — see the class description.
|
|
5197
|
+
*
|
|
5198
|
+
* Everything here is driven by the bucket timer, because the update tick is neither evenly spaced
|
|
5199
|
+
* nor long enough for every client to have reported. Counting on it would compare windows of
|
|
5200
|
+
* different lengths and call the difference a signal.
|
|
5201
|
+
*/
|
|
5202
|
+
update() {
|
|
5203
|
+
}
|
|
5204
|
+
_closeBucket(observedAt) {
|
|
5205
|
+
const totalClients = this._observer.numberOfClients;
|
|
5206
|
+
const affectedClientIds = [...new Set(Array.from(this._trackedIssues, (issue) => issue.clientId))];
|
|
5207
|
+
const affectedCallIds = [...new Set(Array.from(this._trackedIssues, (issue) => issue.callId))];
|
|
5208
|
+
const congestedClients = affectedClientIds.length;
|
|
5209
|
+
const congestedClientRatio = 0 < totalClients ? congestedClients / totalClients : 0;
|
|
5210
|
+
this._history.push({
|
|
5211
|
+
observedAt,
|
|
5212
|
+
totalClients,
|
|
5213
|
+
congestedClients,
|
|
5214
|
+
congestedClientRatio,
|
|
5215
|
+
affectedClientIds,
|
|
5216
|
+
affectedCallIds
|
|
5217
|
+
});
|
|
5218
|
+
while (this._config.historySize < this._history.length) this._history.shift();
|
|
5219
|
+
this._trackedIssues.clear();
|
|
5220
|
+
}
|
|
5221
|
+
/**
|
|
5222
|
+
* Evaluate only the latest completed bucket (the candidate) against the buckets before it (the
|
|
5223
|
+
* baseline) — never against itself. Reached once per newly-closed bucket via {@link update}; the
|
|
5224
|
+
* identity check below additionally guards against evaluating the same bucket twice, in case
|
|
5225
|
+
* `update()` is ever called again before the next rotation.
|
|
5226
|
+
*/
|
|
5227
|
+
_evaluateLatestBucket() {
|
|
5228
|
+
const candidate = this._history[this._history.length - 1];
|
|
5229
|
+
if (candidate === this._lastEvaluatedBucket) return;
|
|
5230
|
+
this._lastEvaluatedBucket = candidate;
|
|
5231
|
+
const baseline = this._history.slice(0, -1);
|
|
5232
|
+
const evaluation = this._evaluateBucket(candidate, baseline);
|
|
5233
|
+
if (!evaluation.isCongested) return;
|
|
5234
|
+
const payload = {
|
|
5235
|
+
affectedCallIds: candidate.affectedCallIds,
|
|
5236
|
+
affectedClientIds: candidate.affectedClientIds,
|
|
5237
|
+
congestedClientRatio: candidate.congestedClientRatio,
|
|
5238
|
+
totalNumberOfClients: candidate.totalClients,
|
|
5239
|
+
numberOfCongestedClients: candidate.congestedClients,
|
|
5240
|
+
historySize: this._history.length,
|
|
5241
|
+
baselineCongestedClientRatio: evaluation.baselineCongestedClientRatio,
|
|
5242
|
+
robustZ: evaluation.robustZ,
|
|
5243
|
+
absoluteIncrease: evaluation.absoluteIncrease,
|
|
5244
|
+
relativeIncrease: evaluation.relativeIncrease
|
|
5245
|
+
};
|
|
5246
|
+
this._observer.emit("observer-issue", {
|
|
5247
|
+
issue: {
|
|
5248
|
+
type: this._config.emittedObserverIssueType,
|
|
5249
|
+
timestamp: Date.now(),
|
|
5250
|
+
payload
|
|
5251
|
+
},
|
|
5252
|
+
observer: this._observer
|
|
5253
|
+
});
|
|
5254
|
+
}
|
|
5255
|
+
/**
|
|
5256
|
+
* Is `candidate` — the latest completed bucket — abnormally high compared with the `baseline`
|
|
5257
|
+
* buckets before it?
|
|
5258
|
+
*
|
|
5259
|
+
* Requires both **statistical** significance (a robust z-score against a median+MAD baseline —
|
|
5260
|
+
* deliberately not Mann-Kendall, which asks "is this a monotonic trend", not "is the latest point
|
|
5261
|
+
* an outlier"; a single sudden spike on an otherwise flat series is exactly what should trigger
|
|
5262
|
+
* here and exactly what a trend test would miss) and **practical** significance (enough affected
|
|
5263
|
+
* clients, and a big enough absolute/relative jump — a statistically significant move in a tiny
|
|
5264
|
+
* or trivial ratio is not worth an alert).
|
|
5265
|
+
*/
|
|
5266
|
+
_evaluateBucket(candidate, baseline) {
|
|
5267
|
+
const baselineRatios = baseline.map((bucket) => bucket.congestedClientRatio);
|
|
5268
|
+
const baselineMedian = median(baselineRatios) ?? 0;
|
|
5269
|
+
const robustZ = robustZScore(candidate.congestedClientRatio, baselineRatios) ?? 0;
|
|
5270
|
+
const absoluteIncrease = candidate.congestedClientRatio - baselineMedian;
|
|
5271
|
+
const relativeIncrease = 0 < baselineMedian ? candidate.congestedClientRatio / baselineMedian : 0 < candidate.congestedClientRatio ? Infinity : 0;
|
|
5272
|
+
const isCongested = this._config.minHistorySize <= baseline.length + 1 && this._config.minAffectedClients <= candidate.congestedClients && this._config.minAbsoluteRatioIncrease <= absoluteIncrease && this._config.minRelativeRatioIncrease <= relativeIncrease && this._config.robustZThreshold <= robustZ;
|
|
5273
|
+
return { isCongested, baselineCongestedClientRatio: baselineMedian, robustZ, absoluteIncrease, relativeIncrease };
|
|
5274
|
+
}
|
|
5275
|
+
};
|
|
5276
|
+
|
|
5277
|
+
// src/detectors/ObserverConcurrentIssueDetector.ts
|
|
5278
|
+
var ObserverConcurrentIssueTypes = {
|
|
5279
|
+
/**
|
|
5280
|
+
* The same issue is open in **several unrelated calls at once**. Those clients share no meeting,
|
|
5281
|
+
* no publisher and no room — only the infrastructure serving them.
|
|
5282
|
+
*/
|
|
5283
|
+
crossCallConcurrentIssues: "CROSS_CALL_CONCURRENT_ISSUES",
|
|
5284
|
+
/** The cross-call version that also started together. The strongest "it's us" signal available. */
|
|
5285
|
+
crossCallIssueOnsetBurst: "CROSS_CALL_ISSUE_ONSET_BURST"
|
|
5286
|
+
};
|
|
5287
|
+
var ObserverConcurrentIssueDetector = class _ObserverConcurrentIssueDetector {
|
|
5288
|
+
constructor(_observer, config = {}) {
|
|
5289
|
+
this._observer = _observer;
|
|
5290
|
+
this._config = {
|
|
5291
|
+
issueTypes: [],
|
|
5292
|
+
minAffectedClients: 3,
|
|
5293
|
+
minAffectedCalls: 2,
|
|
5294
|
+
affectedCallRatioThreshold: 0,
|
|
5295
|
+
onsetBurstWindowInMs: 2e3,
|
|
5296
|
+
cooldownMs: 6e4,
|
|
5297
|
+
...config
|
|
5298
|
+
};
|
|
5299
|
+
for (const type of this._config.issueTypes) {
|
|
5300
|
+
this._observer.activeIssuesRegistry.addIssueTracker(type, this);
|
|
5301
|
+
}
|
|
5302
|
+
}
|
|
5303
|
+
_observer;
|
|
5304
|
+
static NAME = "observer-concurrent-issue-detector";
|
|
5305
|
+
name = _ObserverConcurrentIssueDetector.NAME;
|
|
5306
|
+
_config;
|
|
5307
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5308
|
+
/** issue type -> the issues of that type currently open anywhere in the fleet. */
|
|
5309
|
+
_byType = /* @__PURE__ */ new Map();
|
|
5310
|
+
_size = 0;
|
|
5311
|
+
/** The groups that qualified on the most recent `update()`. Exposed for tests/dashboards. */
|
|
5312
|
+
lastGroups = [];
|
|
5313
|
+
get size() {
|
|
5314
|
+
return this._size;
|
|
5315
|
+
}
|
|
5316
|
+
has(issue) {
|
|
5317
|
+
return this._byType.get(issue.type)?.has(issue) ?? false;
|
|
5318
|
+
}
|
|
5319
|
+
add(issue) {
|
|
5320
|
+
let bucket = this._byType.get(issue.type);
|
|
5321
|
+
if (!bucket) {
|
|
5322
|
+
bucket = /* @__PURE__ */ new Set();
|
|
5323
|
+
this._byType.set(issue.type, bucket);
|
|
5324
|
+
}
|
|
5325
|
+
if (bucket.has(issue)) return;
|
|
5326
|
+
bucket.add(issue);
|
|
5327
|
+
++this._size;
|
|
5328
|
+
}
|
|
5329
|
+
delete(issue) {
|
|
5330
|
+
const bucket = this._byType.get(issue.type);
|
|
5331
|
+
if (!bucket?.delete(issue)) return false;
|
|
5332
|
+
--this._size;
|
|
5333
|
+
if (bucket.size === 0) this._byType.delete(issue.type);
|
|
5334
|
+
return true;
|
|
5335
|
+
}
|
|
5336
|
+
clear() {
|
|
5337
|
+
this._byType.clear();
|
|
5338
|
+
this._size = 0;
|
|
5339
|
+
this.lastGroups = [];
|
|
5340
|
+
}
|
|
5341
|
+
update() {
|
|
5342
|
+
this.lastGroups = [];
|
|
5343
|
+
if (this._size === 0) return;
|
|
5344
|
+
const now = Date.now();
|
|
5345
|
+
for (const [type, issues] of this._byType) {
|
|
5346
|
+
const group = this._groupOf(type, issues);
|
|
5347
|
+
if (group.clientIds.length < this._config.minAffectedClients) continue;
|
|
5348
|
+
if (group.callIds.length < this._config.minAffectedCalls) continue;
|
|
5349
|
+
if (group.affectedCallRatio < this._config.affectedCallRatioThreshold) continue;
|
|
5350
|
+
this.lastGroups.push(group);
|
|
5351
|
+
if (now - (this._lastRaisedAt.get(type) ?? 0) < this._config.cooldownMs) continue;
|
|
5352
|
+
this._lastRaisedAt.set(type, now);
|
|
5353
|
+
const burst = group.onsetSpreadInMs <= this._config.onsetBurstWindowInMs;
|
|
5354
|
+
const issueType = burst ? ObserverConcurrentIssueTypes.crossCallIssueOnsetBurst : ObserverConcurrentIssueTypes.crossCallConcurrentIssues;
|
|
5355
|
+
const conclusion = concludeObserverIssue({
|
|
5356
|
+
issueType: type,
|
|
5357
|
+
affectedClients: group.clientIds.length,
|
|
5358
|
+
totalClients: group.totalClients,
|
|
5359
|
+
affectedCalls: group.callIds.length,
|
|
5360
|
+
totalCalls: group.totalCalls,
|
|
5361
|
+
onsetBurst: burst
|
|
5362
|
+
});
|
|
5363
|
+
this._observer.addIssue({
|
|
5364
|
+
type: issueType,
|
|
5365
|
+
timestamp: now,
|
|
5366
|
+
payload: {
|
|
5367
|
+
type: issueType,
|
|
5368
|
+
issueType: type,
|
|
5369
|
+
scope: "observer",
|
|
5370
|
+
conclusion,
|
|
5371
|
+
clients: group.totalClients,
|
|
5372
|
+
affectedClients: group.clientIds.length,
|
|
5373
|
+
affectedRatio: group.affectedRatio,
|
|
5374
|
+
affectedClientIds: group.clientIds,
|
|
5375
|
+
// The dimension that makes this finding what it is.
|
|
5376
|
+
calls: group.totalCalls,
|
|
5377
|
+
affectedCalls: group.callIds.length,
|
|
5378
|
+
affectedCallRatio: group.affectedCallRatio,
|
|
5379
|
+
affectedCallIds: group.callIds,
|
|
5380
|
+
perCall: group.perCall,
|
|
5381
|
+
onsetSpreadInMs: group.onsetSpreadInMs,
|
|
5382
|
+
onsetBurst: burst,
|
|
5383
|
+
firstObservedAt: group.firstObservedAt
|
|
5384
|
+
}
|
|
5385
|
+
});
|
|
5386
|
+
}
|
|
5387
|
+
}
|
|
5388
|
+
close() {
|
|
5389
|
+
this._observer.activeIssuesRegistry.removeIssueTracker(this);
|
|
5390
|
+
this._lastRaisedAt.clear();
|
|
5391
|
+
this.clear();
|
|
5392
|
+
}
|
|
5393
|
+
_groupOf(type, issues) {
|
|
5394
|
+
const clientIds = /* @__PURE__ */ new Set();
|
|
5395
|
+
const affectedByCall = /* @__PURE__ */ new Map();
|
|
5396
|
+
const list = [];
|
|
5397
|
+
let earliest = Infinity;
|
|
5398
|
+
let latest = -Infinity;
|
|
5399
|
+
for (const issue of issues) {
|
|
5400
|
+
list.push(issue);
|
|
5401
|
+
clientIds.add(issue.clientId);
|
|
5402
|
+
let clients = affectedByCall.get(issue.callId);
|
|
5403
|
+
if (!clients) {
|
|
5404
|
+
clients = /* @__PURE__ */ new Set();
|
|
5405
|
+
affectedByCall.set(issue.callId, clients);
|
|
5406
|
+
}
|
|
5407
|
+
clients.add(issue.clientId);
|
|
5408
|
+
if (issue.observedAt < earliest) earliest = issue.observedAt;
|
|
5409
|
+
if (latest < issue.observedAt) latest = issue.observedAt;
|
|
5410
|
+
}
|
|
5411
|
+
const totalClients = this._observer.numberOfClients;
|
|
5412
|
+
const totalCalls = this._observer.observedCalls.size;
|
|
5413
|
+
const perCall = [];
|
|
5414
|
+
for (const [callId, clients] of affectedByCall) {
|
|
5415
|
+
perCall.push({
|
|
5416
|
+
callId,
|
|
5417
|
+
affectedClients: clients.size,
|
|
5418
|
+
// A call that closed while its issues were still open has no participant count left;
|
|
5419
|
+
// fall back to the affected count rather than reporting a 0 denominator.
|
|
5420
|
+
totalClients: this._observer.observedCalls.get(callId)?.observedClients.size ?? clients.size
|
|
5421
|
+
});
|
|
5422
|
+
}
|
|
5423
|
+
perCall.sort((a, b) => b.affectedClients - a.affectedClients);
|
|
5424
|
+
return {
|
|
5425
|
+
type,
|
|
5426
|
+
issues: list,
|
|
5427
|
+
clientIds: [...clientIds],
|
|
5428
|
+
totalClients,
|
|
5429
|
+
affectedRatio: 0 < totalClients ? clientIds.size / totalClients : 0,
|
|
5430
|
+
callIds: perCall.map((entry) => entry.callId),
|
|
5431
|
+
totalCalls,
|
|
5432
|
+
affectedCallRatio: 0 < totalCalls ? perCall.length / totalCalls : 0,
|
|
5433
|
+
perCall,
|
|
5434
|
+
onsetSpreadInMs: latest - earliest,
|
|
5435
|
+
firstObservedAt: earliest
|
|
5436
|
+
};
|
|
5437
|
+
}
|
|
5438
|
+
};
|
|
5439
|
+
|
|
5440
|
+
// src/detectors/ClientPopulationIssueDetector.ts
|
|
5441
|
+
var ClientPopulationIssueTypes = {
|
|
5442
|
+
/** One issue type is concentrated on one client population while the rest of the fleet is fine. */
|
|
5443
|
+
clientPopulationIssue: "CLIENT_POPULATION_ISSUE"
|
|
5444
|
+
};
|
|
5445
|
+
var ClientPopulationIssueDetector = class _ClientPopulationIssueDetector {
|
|
5446
|
+
constructor(_observer, config = {}) {
|
|
5447
|
+
this._observer = _observer;
|
|
5448
|
+
this._config = {
|
|
5449
|
+
issueTypes: [],
|
|
5450
|
+
groupBy: "browser",
|
|
5451
|
+
includeVersion: true,
|
|
5452
|
+
minPopulationSize: 20,
|
|
5453
|
+
minAffectedClients: 5,
|
|
5454
|
+
affectedRatioThreshold: 0.3,
|
|
5455
|
+
minRelativeRisk: 3,
|
|
5456
|
+
minControlSize: 20,
|
|
5457
|
+
cooldownMs: 3e5,
|
|
5458
|
+
...config
|
|
5459
|
+
};
|
|
5460
|
+
for (const type of this._config.issueTypes) {
|
|
5461
|
+
this._observer.activeIssuesRegistry.addIssueTracker(type, this);
|
|
5462
|
+
}
|
|
5463
|
+
}
|
|
5464
|
+
_observer;
|
|
5465
|
+
static NAME = "client-population-issue-detector";
|
|
5466
|
+
name = _ClientPopulationIssueDetector.NAME;
|
|
5467
|
+
_config;
|
|
5468
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5469
|
+
_issues = /* @__PURE__ */ new Set();
|
|
5470
|
+
/** The populations that qualified on the most recent `update()`. Exposed for tests/dashboards. */
|
|
5471
|
+
lastPopulations = [];
|
|
5472
|
+
get size() {
|
|
5473
|
+
return this._issues.size;
|
|
5474
|
+
}
|
|
5475
|
+
has(issue) {
|
|
5476
|
+
return this._issues.has(issue);
|
|
5477
|
+
}
|
|
5478
|
+
add(issue) {
|
|
5479
|
+
this._issues.add(issue);
|
|
5480
|
+
}
|
|
5481
|
+
delete(issue) {
|
|
5482
|
+
return this._issues.delete(issue);
|
|
5483
|
+
}
|
|
5484
|
+
clear() {
|
|
5485
|
+
this._issues.clear();
|
|
5486
|
+
this.lastPopulations = [];
|
|
5487
|
+
}
|
|
5488
|
+
update() {
|
|
5489
|
+
this.lastPopulations = [];
|
|
5490
|
+
if (this._issues.size === 0) return;
|
|
5491
|
+
const now = Date.now();
|
|
5492
|
+
const populationSizes = /* @__PURE__ */ new Map();
|
|
5493
|
+
for (const call of this._observer.observedCalls.values()) {
|
|
5494
|
+
for (const client of call.observedClients.values()) {
|
|
5495
|
+
const population = this._populationOf(client);
|
|
5496
|
+
if (population === void 0) continue;
|
|
5497
|
+
populationSizes.set(population, (populationSizes.get(population) ?? 0) + 1);
|
|
5498
|
+
}
|
|
5499
|
+
}
|
|
5500
|
+
if (populationSizes.size === 0) return;
|
|
5501
|
+
const affected = /* @__PURE__ */ new Map();
|
|
5502
|
+
for (const issue of this._issues) {
|
|
5503
|
+
const client = this._observer.observedCalls.get(issue.callId)?.observedClients.get(issue.clientId);
|
|
5504
|
+
if (!client) continue;
|
|
5505
|
+
const population = this._populationOf(client);
|
|
5506
|
+
if (population === void 0) continue;
|
|
5507
|
+
let byPopulation = affected.get(issue.type);
|
|
5508
|
+
if (!byPopulation) {
|
|
5509
|
+
byPopulation = /* @__PURE__ */ new Map();
|
|
5510
|
+
affected.set(issue.type, byPopulation);
|
|
5511
|
+
}
|
|
5512
|
+
const clientIds = byPopulation.get(population) ?? /* @__PURE__ */ new Set();
|
|
5513
|
+
clientIds.add(issue.clientId);
|
|
5514
|
+
byPopulation.set(population, clientIds);
|
|
5515
|
+
}
|
|
5516
|
+
for (const [issueType, byPopulation] of affected) {
|
|
5517
|
+
let totalAffected = 0;
|
|
5518
|
+
for (const clientIds of byPopulation.values()) totalAffected += clientIds.size;
|
|
5519
|
+
let totalClients = 0;
|
|
5520
|
+
for (const size of populationSizes.values()) totalClients += size;
|
|
5521
|
+
for (const [population, clientIds] of byPopulation) {
|
|
5522
|
+
const clients = populationSizes.get(population) ?? clientIds.size;
|
|
5523
|
+
const rollup = this._rollupOf(
|
|
5524
|
+
population,
|
|
5525
|
+
issueType,
|
|
5526
|
+
clientIds,
|
|
5527
|
+
clients,
|
|
5528
|
+
totalClients - clients,
|
|
5529
|
+
totalAffected - clientIds.size
|
|
5530
|
+
);
|
|
5531
|
+
if (rollup.clients < this._config.minPopulationSize) continue;
|
|
5532
|
+
if (rollup.affectedClients < this._config.minAffectedClients) continue;
|
|
5533
|
+
if (rollup.affectedRatio < this._config.affectedRatioThreshold) continue;
|
|
5534
|
+
if (rollup.controlClients < this._config.minControlSize) continue;
|
|
5535
|
+
if (rollup.relativeRisk < this._config.minRelativeRisk) continue;
|
|
5536
|
+
this.lastPopulations.push(rollup);
|
|
5537
|
+
const key = `${population}:${issueType}`;
|
|
5538
|
+
if (now - (this._lastRaisedAt.get(key) ?? 0) < this._config.cooldownMs) continue;
|
|
5539
|
+
this._lastRaisedAt.set(key, now);
|
|
5540
|
+
this._observer.addIssue({
|
|
5541
|
+
type: ClientPopulationIssueTypes.clientPopulationIssue,
|
|
5542
|
+
timestamp: now,
|
|
5543
|
+
payload: {
|
|
5544
|
+
type: ClientPopulationIssueTypes.clientPopulationIssue,
|
|
5545
|
+
...rollup,
|
|
5546
|
+
conclusion: {
|
|
5547
|
+
faultDomain: "client-population",
|
|
5548
|
+
summary: `'${issueType}' is ${this._riskText(rollup.relativeRisk)} more likely on ${population} than on the rest of the fleet (${rollup.affectedClients}/${rollup.clients} vs ${rollup.controlAffectedClients}/${rollup.controlClients})`,
|
|
5549
|
+
recommendation: "this is not an SFU symptom \u2014 look at what those clients share: a recent release, a browser version, or shared/virtualised hardware",
|
|
5550
|
+
confidence: this._confidenceOf(rollup)
|
|
5551
|
+
}
|
|
5552
|
+
}
|
|
5553
|
+
});
|
|
5554
|
+
}
|
|
5555
|
+
}
|
|
5556
|
+
}
|
|
5557
|
+
close() {
|
|
5558
|
+
this._observer.activeIssuesRegistry.removeIssueTracker(this);
|
|
5559
|
+
this._lastRaisedAt.clear();
|
|
5560
|
+
this.clear();
|
|
5561
|
+
}
|
|
5562
|
+
/** `undefined` when the client never reported this attribute — see the class description. */
|
|
5563
|
+
_populationOf(client) {
|
|
5564
|
+
const attribute = client[this._config.groupBy];
|
|
5565
|
+
if (!attribute?.name) return void 0;
|
|
5566
|
+
return this._config.includeVersion && attribute.version ? `${attribute.name} ${attribute.version}` : attribute.name;
|
|
5567
|
+
}
|
|
5568
|
+
_rollupOf(population, issueType, clientIds, clients, controlClients, controlAffectedClients) {
|
|
5569
|
+
const affectedRatio = 0 < clients ? clientIds.size / clients : 0;
|
|
5570
|
+
const controlAffectedRatio = 0 < controlClients ? controlAffectedClients / controlClients : 0;
|
|
5571
|
+
return {
|
|
5572
|
+
population,
|
|
5573
|
+
axis: this._config.groupBy,
|
|
5574
|
+
issueType,
|
|
5575
|
+
clients,
|
|
5576
|
+
affectedClients: clientIds.size,
|
|
5577
|
+
affectedRatio,
|
|
5578
|
+
affectedClientIds: [...clientIds],
|
|
5579
|
+
controlClients,
|
|
5580
|
+
controlAffectedClients,
|
|
5581
|
+
controlAffectedRatio,
|
|
5582
|
+
// A spotless control group means nobody outside this population has the problem at all.
|
|
5583
|
+
// `Infinity` says that plainly rather than dividing by zero or inventing a ceiling.
|
|
5584
|
+
relativeRisk: 0 < controlAffectedRatio ? affectedRatio / controlAffectedRatio : 0 < affectedRatio ? Infinity : 0
|
|
5585
|
+
};
|
|
5586
|
+
}
|
|
5587
|
+
_riskText(relativeRisk) {
|
|
5588
|
+
return Number.isFinite(relativeRisk) ? `${relativeRisk.toFixed(1)}x` : "exclusively";
|
|
5589
|
+
}
|
|
5590
|
+
/** Bigger populations and starker contrasts are harder to produce by chance. */
|
|
5591
|
+
_confidenceOf(rollup) {
|
|
5592
|
+
let confidence = 0.4;
|
|
5593
|
+
if (100 <= rollup.clients) confidence += 0.2;
|
|
5594
|
+
else if (50 <= rollup.clients) confidence += 0.1;
|
|
5595
|
+
if (10 <= rollup.relativeRisk) confidence += 0.3;
|
|
5596
|
+
else if (5 <= rollup.relativeRisk) confidence += 0.2;
|
|
5597
|
+
else confidence += 0.1;
|
|
5598
|
+
return Math.min(1, Math.round(confidence * 100) / 100);
|
|
5599
|
+
}
|
|
5600
|
+
};
|
|
5601
|
+
|
|
5602
|
+
// src/detectors/TurnServerHealthDetector.ts
|
|
5603
|
+
var TurnServerHealthTypes = {
|
|
5604
|
+
/** One TURN server's clients are in trouble while other servers' clients are fine. */
|
|
5605
|
+
turnServerDegraded: "TURN_SERVER_DEGRADED"
|
|
5606
|
+
};
|
|
5607
|
+
var TurnServerHealthDetector = class _TurnServerHealthDetector {
|
|
5608
|
+
constructor(_observer, config = {}) {
|
|
5609
|
+
this._observer = _observer;
|
|
5610
|
+
this._config = {
|
|
5611
|
+
minClientsPerServer: 5,
|
|
5612
|
+
degradedRatioThreshold: 0.5,
|
|
5613
|
+
// any open issue: the question is where trouble clusters, not what it is
|
|
5614
|
+
issueTypes: [],
|
|
5615
|
+
consecutiveTicks: 2,
|
|
5616
|
+
cooldownMs: 6e4,
|
|
5617
|
+
...config
|
|
5618
|
+
};
|
|
5619
|
+
}
|
|
5620
|
+
_observer;
|
|
5621
|
+
static NAME = "turn-server-health-detector";
|
|
5622
|
+
name = _TurnServerHealthDetector.NAME;
|
|
5623
|
+
_config;
|
|
5624
|
+
_streaks = /* @__PURE__ */ new Map();
|
|
5625
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5626
|
+
/** The per-server rollup computed on the most recent `update()`. */
|
|
5627
|
+
lastServers = [];
|
|
5628
|
+
update() {
|
|
5629
|
+
const now = Date.now();
|
|
5630
|
+
const wanted = new Set(this._config.issueTypes);
|
|
5631
|
+
const issuesByClientId = /* @__PURE__ */ new Map();
|
|
5632
|
+
for (const issue of this._observer.activeIssuesRegistry.values()) {
|
|
5633
|
+
if (0 < wanted.size && !wanted.has(issue.type)) continue;
|
|
5634
|
+
const types = issuesByClientId.get(issue.clientId);
|
|
5635
|
+
if (types) types.push(issue.type);
|
|
5636
|
+
else issuesByClientId.set(issue.clientId, [issue.type]);
|
|
5637
|
+
}
|
|
5638
|
+
this.lastServers = [...this._observer.observedTURN.servers].map(([serverUrl, server]) => this._serverHealth(serverUrl, server.observedPeerConnections, issuesByClientId));
|
|
5639
|
+
for (const health of this.lastServers) {
|
|
5640
|
+
const serverUrl = health.serverUrl;
|
|
5641
|
+
if (health.clients < this._config.minClientsPerServer || health.degradedRatio < this._config.degradedRatioThreshold) {
|
|
5642
|
+
this._streaks.delete(serverUrl);
|
|
5643
|
+
continue;
|
|
5644
|
+
}
|
|
5645
|
+
const ticks = (this._streaks.get(serverUrl) ?? 0) + 1;
|
|
5646
|
+
this._streaks.set(serverUrl, ticks);
|
|
5647
|
+
if (ticks < this._config.consecutiveTicks) continue;
|
|
5648
|
+
if (now - (this._lastRaisedAt.get(serverUrl) ?? 0) < this._config.cooldownMs) continue;
|
|
5649
|
+
this._lastRaisedAt.set(serverUrl, now);
|
|
5650
|
+
const otherServers = this.lastServers.filter((server) => server.serverUrl !== serverUrl).map((server) => ({ serverUrl: server.serverUrl, clients: server.clients, degradedRatio: server.degradedRatio }));
|
|
5651
|
+
this._observer.addIssue({
|
|
5652
|
+
type: TurnServerHealthTypes.turnServerDegraded,
|
|
5653
|
+
timestamp: now,
|
|
5654
|
+
payload: { type: TurnServerHealthTypes.turnServerDegraded, ...health, otherServers }
|
|
5655
|
+
});
|
|
5656
|
+
}
|
|
5657
|
+
for (const serverUrl of [...this._streaks.keys()]) {
|
|
5658
|
+
if (!this.lastServers.some((server) => server.serverUrl === serverUrl)) this._streaks.delete(serverUrl);
|
|
5659
|
+
}
|
|
5660
|
+
}
|
|
5661
|
+
close() {
|
|
5662
|
+
this._streaks.clear();
|
|
5663
|
+
this._lastRaisedAt.clear();
|
|
5664
|
+
this.lastServers = [];
|
|
5665
|
+
}
|
|
5666
|
+
_serverHealth(serverUrl, peerConnections, issuesByClientId) {
|
|
5667
|
+
const clientIds = /* @__PURE__ */ new Set();
|
|
5668
|
+
const affectedClientIds = /* @__PURE__ */ new Set();
|
|
5669
|
+
const typeCounts = /* @__PURE__ */ new Map();
|
|
5670
|
+
for (const peerConnection of peerConnections.values()) {
|
|
5671
|
+
const clientId = peerConnection.client.clientId;
|
|
5672
|
+
clientIds.add(clientId);
|
|
5673
|
+
const types = issuesByClientId.get(clientId);
|
|
5674
|
+
if (!types) continue;
|
|
5675
|
+
affectedClientIds.add(clientId);
|
|
5676
|
+
for (const type of types) typeCounts.set(type, (typeCounts.get(type) ?? 0) + 1);
|
|
5677
|
+
}
|
|
5678
|
+
return {
|
|
5679
|
+
serverUrl,
|
|
5680
|
+
clients: clientIds.size,
|
|
5681
|
+
degradedClients: affectedClientIds.size,
|
|
5682
|
+
degradedRatio: 0 < clientIds.size ? affectedClientIds.size / clientIds.size : 0,
|
|
5683
|
+
affectedClientIds: [...affectedClientIds],
|
|
5684
|
+
issueTypes: [...typeCounts.entries()].sort((a, b) => b[1] - a[1]).map(([type]) => type)
|
|
5685
|
+
};
|
|
5686
|
+
}
|
|
5687
|
+
};
|
|
5688
|
+
|
|
5689
|
+
// src/detectors/TurnServerOutageDetector.ts
|
|
5690
|
+
var TurnServerOutageTypes = {
|
|
5691
|
+
/** One TURN server's relayed population collapsed while the rest of the fleet is fine. */
|
|
5692
|
+
turnServerOutage: "TURN_SERVER_OUTAGE"
|
|
5693
|
+
};
|
|
5694
|
+
var brokenStates = /* @__PURE__ */ new Set(["disconnected", "failed", "closed"]);
|
|
5695
|
+
var TurnServerOutageDetector = class _TurnServerOutageDetector {
|
|
5696
|
+
constructor(_observer, config = {}) {
|
|
5697
|
+
this._observer = _observer;
|
|
5698
|
+
this._config = {
|
|
5699
|
+
minClientsAtPeak: 5,
|
|
5700
|
+
// an outage is near-total by definition; partial degradation is the health detector's question
|
|
5701
|
+
lossRatioThreshold: 0.8,
|
|
5702
|
+
peakWindowMs: 12e4,
|
|
5703
|
+
// absence without a control group is just quiet
|
|
5704
|
+
requireControlGroup: true,
|
|
5705
|
+
minControlGroupClients: 5,
|
|
5706
|
+
controlGroupHealthyRatio: 0.7,
|
|
5707
|
+
consecutiveTicks: 2,
|
|
5708
|
+
// one event, not one per tick
|
|
5709
|
+
cooldownMs: 3e5,
|
|
5710
|
+
...config
|
|
5711
|
+
};
|
|
5712
|
+
}
|
|
5713
|
+
_observer;
|
|
5714
|
+
static NAME = "turn-server-outage-detector";
|
|
5715
|
+
name = _TurnServerOutageDetector.NAME;
|
|
5716
|
+
_config;
|
|
5717
|
+
/** serverUrl -> recent population observations, used to derive the windowed peak. */
|
|
5718
|
+
_peaks = /* @__PURE__ */ new Map();
|
|
5719
|
+
_streaks = /* @__PURE__ */ new Map();
|
|
5720
|
+
_lastRaisedAt = /* @__PURE__ */ new Map();
|
|
5721
|
+
update() {
|
|
5722
|
+
const now = Date.now();
|
|
5723
|
+
const servers = [...this._observer.observedTURN.servers];
|
|
5724
|
+
const populations = /* @__PURE__ */ new Map();
|
|
5725
|
+
for (const [serverUrl, server] of servers) {
|
|
5726
|
+
populations.set(serverUrl, this._populationOf(server.observedPeerConnections));
|
|
5727
|
+
}
|
|
5728
|
+
for (const [serverUrl, population] of populations) {
|
|
5729
|
+
const live = population.healthy.size;
|
|
5730
|
+
const broken = population.broken.size;
|
|
5731
|
+
const peak = this._recordAndPeak(serverUrl, live + broken, now);
|
|
5732
|
+
if (peak < this._config.minClientsAtPeak) {
|
|
5733
|
+
this._streaks.delete(serverUrl);
|
|
5734
|
+
continue;
|
|
5735
|
+
}
|
|
5736
|
+
const lost = Math.max(0, peak - live);
|
|
5737
|
+
const lossRatio = lost / peak;
|
|
5738
|
+
if (lossRatio < this._config.lossRatioThreshold) {
|
|
5739
|
+
this._streaks.delete(serverUrl);
|
|
5740
|
+
continue;
|
|
5741
|
+
}
|
|
5742
|
+
const control = this._controlGroup(serverUrl, populations);
|
|
5743
|
+
if (this._config.requireControlGroup) {
|
|
5744
|
+
if (control.total < this._config.minControlGroupClients) {
|
|
5745
|
+
this._streaks.delete(serverUrl);
|
|
5746
|
+
continue;
|
|
5747
|
+
}
|
|
5748
|
+
if (control.healthyRatio < this._config.controlGroupHealthyRatio) {
|
|
5749
|
+
this._streaks.delete(serverUrl);
|
|
5750
|
+
continue;
|
|
5751
|
+
}
|
|
5752
|
+
}
|
|
5753
|
+
const ticks = (this._streaks.get(serverUrl) ?? 0) + 1;
|
|
5754
|
+
this._streaks.set(serverUrl, ticks);
|
|
5755
|
+
if (ticks < this._config.consecutiveTicks) continue;
|
|
5756
|
+
if (now - (this._lastRaisedAt.get(serverUrl) ?? 0) < this._config.cooldownMs) continue;
|
|
5757
|
+
this._lastRaisedAt.set(serverUrl, now);
|
|
5758
|
+
this._observer.addIssue({
|
|
5759
|
+
type: TurnServerOutageTypes.turnServerOutage,
|
|
5760
|
+
timestamp: now,
|
|
5761
|
+
payload: {
|
|
5762
|
+
type: TurnServerOutageTypes.turnServerOutage,
|
|
5763
|
+
serverUrl,
|
|
5764
|
+
peakClients: peak,
|
|
5765
|
+
currentClients: live,
|
|
5766
|
+
disruptedClients: broken,
|
|
5767
|
+
lostClients: lost,
|
|
5768
|
+
lossRatio,
|
|
5769
|
+
disruptedClientIds: [...population.broken],
|
|
5770
|
+
controlGroupClients: control.total,
|
|
5771
|
+
controlGroupHealthyClients: control.healthy,
|
|
5772
|
+
controlGroupHealthyRatio: control.healthyRatio,
|
|
5773
|
+
peakWindowMs: this._config.peakWindowMs
|
|
5774
|
+
}
|
|
5775
|
+
});
|
|
5776
|
+
}
|
|
5777
|
+
for (const key of [...this._peaks.keys()]) {
|
|
5778
|
+
if (populations.has(key)) continue;
|
|
5779
|
+
this._peaks.delete(key);
|
|
5780
|
+
this._streaks.delete(key);
|
|
5781
|
+
}
|
|
5782
|
+
}
|
|
5783
|
+
close() {
|
|
5784
|
+
this._peaks.clear();
|
|
5785
|
+
this._streaks.clear();
|
|
5786
|
+
this._lastRaisedAt.clear();
|
|
5787
|
+
}
|
|
5788
|
+
/** Distinct clients on a server, split by whether their relayed transport is actually up. */
|
|
5789
|
+
_populationOf(peerConnections) {
|
|
5790
|
+
const healthy = /* @__PURE__ */ new Set();
|
|
5791
|
+
const broken = /* @__PURE__ */ new Set();
|
|
5792
|
+
for (const peerConnection of peerConnections.values()) {
|
|
5793
|
+
const clientId = peerConnection.client.clientId;
|
|
5794
|
+
const isBroken = peerConnection.closed || brokenStates.has(peerConnection.iceConnectionState ?? "") || brokenStates.has(peerConnection.connectionState ?? "");
|
|
5795
|
+
if (isBroken) broken.add(clientId);
|
|
5796
|
+
else healthy.add(clientId);
|
|
5797
|
+
}
|
|
5798
|
+
for (const clientId of healthy) broken.delete(clientId);
|
|
5799
|
+
return { healthy, broken };
|
|
5800
|
+
}
|
|
5801
|
+
/** Record this tick's population and return the peak across `peakWindowMs`. */
|
|
5802
|
+
_recordAndPeak(serverUrl, clients, now) {
|
|
5803
|
+
const entries = this._peaks.get(serverUrl) ?? [];
|
|
5804
|
+
const cutoff = now - this._config.peakWindowMs;
|
|
5805
|
+
const kept = entries.filter((e) => cutoff <= e.at);
|
|
5806
|
+
kept.push({ clients, at: now });
|
|
5807
|
+
this._peaks.set(serverUrl, kept);
|
|
5808
|
+
return kept.reduce((max, e) => Math.max(max, e.clients), 0);
|
|
5809
|
+
}
|
|
5810
|
+
/**
|
|
5811
|
+
* Everyone *not* relayed through `serverUrl`: clients on other TURN servers plus every client
|
|
5812
|
+
* the observer knows about that isn't relayed at all. The healthy share of that group is what
|
|
5813
|
+
* separates "this server broke" from "everything broke".
|
|
5814
|
+
*/
|
|
5815
|
+
_controlGroup(serverUrl, populations) {
|
|
5816
|
+
const healthy = /* @__PURE__ */ new Set();
|
|
5817
|
+
const broken = /* @__PURE__ */ new Set();
|
|
5818
|
+
for (const [otherUrl, population] of populations) {
|
|
5819
|
+
if (otherUrl === serverUrl) continue;
|
|
5820
|
+
for (const clientId of population.healthy) healthy.add(clientId);
|
|
5821
|
+
for (const clientId of population.broken) broken.add(clientId);
|
|
5822
|
+
}
|
|
5823
|
+
const onThisServer = populations.get(serverUrl);
|
|
5824
|
+
for (const call of this._observer.observedCalls.values()) {
|
|
5825
|
+
for (const client of call.observedClients.values()) {
|
|
5826
|
+
if (onThisServer?.healthy.has(client.clientId) || onThisServer?.broken.has(client.clientId)) continue;
|
|
5827
|
+
if (broken.has(client.clientId)) continue;
|
|
5828
|
+
healthy.add(client.clientId);
|
|
5829
|
+
}
|
|
5830
|
+
}
|
|
5831
|
+
for (const clientId of healthy) broken.delete(clientId);
|
|
5832
|
+
const total = healthy.size + broken.size;
|
|
5833
|
+
return {
|
|
5834
|
+
total,
|
|
5835
|
+
healthy: healthy.size,
|
|
5836
|
+
healthyRatio: 0 < total ? healthy.size / total : 0
|
|
5837
|
+
};
|
|
5838
|
+
}
|
|
5839
|
+
};
|
|
5840
|
+
|
|
5841
|
+
// src/utils/SlidingWindow.ts
|
|
5842
|
+
var SlidingWindow = class {
|
|
5843
|
+
constructor(windowMs, maxEntries = 1024) {
|
|
5844
|
+
this.windowMs = windowMs;
|
|
5845
|
+
this.maxEntries = maxEntries;
|
|
5846
|
+
}
|
|
5847
|
+
windowMs;
|
|
5848
|
+
maxEntries;
|
|
5849
|
+
_entries = [];
|
|
5850
|
+
_latest = -Infinity;
|
|
5851
|
+
/** Add an entry (defaults to `Date.now()`), then evict anything outside the window. */
|
|
5852
|
+
add(value, timestamp = Date.now()) {
|
|
5853
|
+
if (!Number.isFinite(timestamp)) return;
|
|
5854
|
+
const entry = { timestamp, value };
|
|
5855
|
+
const last = this._entries[this._entries.length - 1];
|
|
5856
|
+
if (last === void 0 || last.timestamp <= timestamp) {
|
|
5857
|
+
this._entries.push(entry);
|
|
5858
|
+
} else {
|
|
5859
|
+
let index = this._entries.length - 1;
|
|
5860
|
+
while (0 < index && timestamp < this._entries[index - 1].timestamp) --index;
|
|
5861
|
+
this._entries.splice(index, 0, entry);
|
|
5862
|
+
}
|
|
5863
|
+
if (this._latest < timestamp) this._latest = timestamp;
|
|
5864
|
+
this._evict(timestamp);
|
|
5865
|
+
}
|
|
5866
|
+
/**
|
|
5867
|
+
* The entries still inside the window, oldest first.
|
|
5868
|
+
*
|
|
5869
|
+
* A **copy** — callers routinely map/sort what they get back, and handing out the live array let
|
|
5870
|
+
* them mutate the window from the outside.
|
|
5871
|
+
*/
|
|
5872
|
+
entries(now = Date.now()) {
|
|
5873
|
+
this._evict(now);
|
|
5874
|
+
return [...this._entries];
|
|
5875
|
+
}
|
|
5876
|
+
/** The values still inside the window, oldest first. */
|
|
5877
|
+
values(now = Date.now()) {
|
|
5878
|
+
this._evict(now);
|
|
5879
|
+
return this._entries.map((entry) => entry.value);
|
|
5880
|
+
}
|
|
5881
|
+
/**
|
|
5882
|
+
* How many entries are inside the window as of `now`.
|
|
5883
|
+
*
|
|
5884
|
+
* Prefer this to `values(now).length`: counting through {@link values} allocates an array of every
|
|
5885
|
+
* entry only to read its length, which on a hot path is the whole cost of the call.
|
|
5886
|
+
*/
|
|
5887
|
+
count(now = Date.now()) {
|
|
5888
|
+
this._evict(now);
|
|
5889
|
+
return this._entries.length;
|
|
5890
|
+
}
|
|
5891
|
+
/** Retained entries, without evicting first. See {@link count} for the windowed answer. */
|
|
5892
|
+
get size() {
|
|
5893
|
+
return this._entries.length;
|
|
5894
|
+
}
|
|
5895
|
+
clear() {
|
|
5896
|
+
this._entries = [];
|
|
5897
|
+
this._latest = -Infinity;
|
|
5898
|
+
}
|
|
5899
|
+
_evict(now) {
|
|
5900
|
+
if (Number.isFinite(now) && this._latest < now) this._latest = now;
|
|
5901
|
+
const oldest = this._latest - this.windowMs;
|
|
5902
|
+
let expired = 0;
|
|
5903
|
+
while (expired < this._entries.length && this._entries[expired].timestamp < oldest) ++expired;
|
|
5904
|
+
if (0 < expired) this._entries.splice(0, expired);
|
|
5905
|
+
const excess = this._entries.length - this.maxEntries;
|
|
5906
|
+
if (0 < excess) this._entries.splice(0, excess);
|
|
5907
|
+
}
|
|
5908
|
+
};
|
|
5909
|
+
|
|
5910
|
+
// src/validators/SimulcastReceiverValidator.ts
|
|
5911
|
+
var LOWEST_COMMON_DENOMINATOR_ISSUE = "WORST_RECEIVER_CONTAGION";
|
|
5912
|
+
var SimulcastReceiverValidator = class _SimulcastReceiverValidator {
|
|
5913
|
+
constructor(_observer, onDone, config = {}) {
|
|
5914
|
+
this._observer = _observer;
|
|
5915
|
+
this.onDone = onDone;
|
|
5916
|
+
this._config = {
|
|
5917
|
+
minChecks: 3,
|
|
5918
|
+
minReceivers: 3,
|
|
5919
|
+
windowMs: 1e4,
|
|
5920
|
+
minSamples: 5,
|
|
5921
|
+
outlierRatioThreshold: 0.5,
|
|
5922
|
+
trackingRatioThreshold: 0.8,
|
|
5923
|
+
...config
|
|
5924
|
+
};
|
|
5925
|
+
}
|
|
5926
|
+
_observer;
|
|
5927
|
+
onDone;
|
|
5928
|
+
static NAME = "simulcast-receivers";
|
|
5929
|
+
name = _SimulcastReceiverValidator.NAME;
|
|
5930
|
+
startedAt = Date.now();
|
|
5931
|
+
report = { ready: false };
|
|
5932
|
+
_config;
|
|
5933
|
+
_windows = /* @__PURE__ */ new Map();
|
|
5934
|
+
_checks = 0;
|
|
5935
|
+
_done = false;
|
|
5936
|
+
/** How many times the comparison actually ran. `0` means nothing was established. */
|
|
5937
|
+
get checks() {
|
|
5938
|
+
return this._checks;
|
|
5939
|
+
}
|
|
5940
|
+
/** Give up without a verdict, freeing anything waiting on this validator. */
|
|
5941
|
+
cancel(reason = "cancelled") {
|
|
5942
|
+
this._finish({ verdict: "inconclusive", reason });
|
|
5943
|
+
}
|
|
5944
|
+
update() {
|
|
5945
|
+
if (this._done) return;
|
|
5946
|
+
const now = Date.now();
|
|
5947
|
+
const live = /* @__PURE__ */ new Set();
|
|
5948
|
+
for (const call of this._observer.observedCalls.values()) {
|
|
5949
|
+
if (!call.remoteTrackResolver) continue;
|
|
5950
|
+
for (const client of call.observedClients.values()) {
|
|
5951
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
5952
|
+
for (const publisher of peerConnection.observedOutboundTracks.values()) {
|
|
5953
|
+
if (publisher.kind !== "video") continue;
|
|
5954
|
+
live.add(publisher.id);
|
|
5955
|
+
if (this._inspect(call.callId, client.clientId, publisher, now)) return;
|
|
5956
|
+
}
|
|
5957
|
+
}
|
|
5958
|
+
}
|
|
5959
|
+
}
|
|
5960
|
+
for (const trackId of [...this._windows.keys()]) {
|
|
5961
|
+
if (!live.has(trackId)) this._windows.delete(trackId);
|
|
5962
|
+
}
|
|
5963
|
+
}
|
|
5964
|
+
/** Returns `true` when a verdict was reached and the caller should stop iterating. */
|
|
5965
|
+
_inspect(callId, publisherClientId, publisher, now) {
|
|
5966
|
+
const publisherBitrate = publisher.bitrate;
|
|
5967
|
+
if (publisherBitrate === void 0 || publisherBitrate <= 0) return false;
|
|
5968
|
+
if (publisher.remoteInboundTracks.size < this._config.minReceivers) return false;
|
|
5969
|
+
const receivers = [];
|
|
5970
|
+
for (const receiver of publisher.remoteInboundTracks) {
|
|
5971
|
+
const bitrate = receiver.getInboundRtp()?.bitrate;
|
|
5972
|
+
if (bitrate === void 0 || bitrate <= 0) continue;
|
|
5973
|
+
receivers.push({ clientId: receiver.getPeerConnection().client.clientId, bitrate });
|
|
5974
|
+
}
|
|
5975
|
+
if (receivers.length < this._config.minReceivers) return false;
|
|
5976
|
+
const bitrates = receivers.map((receiver) => receiver.bitrate);
|
|
5977
|
+
const worstReceiver = receivers.reduce((a, b) => a.bitrate <= b.bitrate ? a : b);
|
|
5978
|
+
const worst = worstReceiver.bitrate;
|
|
5979
|
+
const median2 = percentile(bitrates, 0.5) ?? worst;
|
|
5980
|
+
const window = this._windowOf(publisher.id);
|
|
5981
|
+
window.add({ publisher: publisherBitrate, worst, median: median2 }, now);
|
|
5982
|
+
const samples = window.values(now);
|
|
5983
|
+
if (samples.length < this._config.minSamples) return false;
|
|
5984
|
+
const latest = samples[samples.length - 1];
|
|
5985
|
+
const outlierRatio = 0 < latest.median ? latest.worst / latest.median : 1;
|
|
5986
|
+
if (this._config.outlierRatioThreshold < outlierRatio) return false;
|
|
5987
|
+
++this._checks;
|
|
5988
|
+
window.clear();
|
|
5989
|
+
const trackingWithWorst = correlation(samples.map((s) => s.publisher), samples.map((s) => s.worst));
|
|
5990
|
+
const trackingWithMedian = correlation(samples.map((s) => s.publisher), samples.map((s) => s.median));
|
|
5991
|
+
const dragged = this._config.trackingRatioThreshold <= trackingWithWorst && trackingWithMedian < trackingWithWorst;
|
|
5992
|
+
const evidence = {
|
|
5993
|
+
callId,
|
|
5994
|
+
trackId: publisher.id,
|
|
5995
|
+
publisherClientId,
|
|
5996
|
+
worstReceiverClientId: worstReceiver.clientId,
|
|
5997
|
+
publisherBitrate,
|
|
5998
|
+
worstReceiverBitrate: latest.worst,
|
|
5999
|
+
medianReceiverBitrate: latest.median,
|
|
6000
|
+
trackingWithWorst,
|
|
6001
|
+
trackingWithMedian
|
|
6002
|
+
};
|
|
6003
|
+
if (dragged) {
|
|
6004
|
+
this._finish({ verdict: "layer-decided-lowest-common-denominator", evidence });
|
|
6005
|
+
return true;
|
|
6006
|
+
}
|
|
6007
|
+
if (this._config.minChecks <= this._checks) {
|
|
6008
|
+
this._finish({ verdict: "layer-decided-per-receiver", evidence });
|
|
6009
|
+
return true;
|
|
6010
|
+
}
|
|
6011
|
+
return false;
|
|
6012
|
+
}
|
|
6013
|
+
/**
|
|
6014
|
+
* Settle on a verdict, exactly once.
|
|
6015
|
+
*
|
|
6016
|
+
* The guard is not paranoia: `onDone` removes this validator from the observer, and a second call
|
|
6017
|
+
* would emit a second `validation-ready` for a validator that is no longer registered — e.g. when
|
|
6018
|
+
* `observer.close()` cancels a validator that decided earlier in the same tick.
|
|
6019
|
+
*/
|
|
6020
|
+
_finish(outcome) {
|
|
6021
|
+
if (this._done) return;
|
|
6022
|
+
this._done = true;
|
|
6023
|
+
const decidedAt = Date.now();
|
|
6024
|
+
this._windows.clear();
|
|
6025
|
+
this.report = {
|
|
6026
|
+
ready: true,
|
|
6027
|
+
decidedAt,
|
|
6028
|
+
startedAt: this.startedAt,
|
|
6029
|
+
checks: this._checks,
|
|
6030
|
+
...outcome
|
|
6031
|
+
};
|
|
6032
|
+
if (outcome.verdict === "layer-decided-lowest-common-denominator") {
|
|
6033
|
+
this._observer.addIssue({
|
|
6034
|
+
type: LOWEST_COMMON_DENOMINATOR_ISSUE,
|
|
6035
|
+
timestamp: decidedAt,
|
|
6036
|
+
payload: {
|
|
6037
|
+
type: LOWEST_COMMON_DENOMINATOR_ISSUE,
|
|
6038
|
+
checks: this._checks,
|
|
6039
|
+
...outcome.evidence,
|
|
6040
|
+
conclusion: {
|
|
6041
|
+
faultDomain: "infrastructure",
|
|
6042
|
+
summary: "one bad receiver is dragging a publisher's bitrate down for everyone \u2014 the SFU is not selecting layers per consumer",
|
|
6043
|
+
recommendation: "check that simulcast/SVC is enabled and layers are chosen per consumer, and that the SFU terminates receiver reports instead of forwarding them; this is a build/config property, not a transient",
|
|
6044
|
+
confidence: 0.8
|
|
6045
|
+
}
|
|
6046
|
+
}
|
|
6047
|
+
});
|
|
6048
|
+
}
|
|
6049
|
+
this.onDone(this.report);
|
|
6050
|
+
}
|
|
6051
|
+
_windowOf(trackId) {
|
|
6052
|
+
let window = this._windows.get(trackId);
|
|
6053
|
+
if (!window) {
|
|
6054
|
+
window = new SlidingWindow(this._config.windowMs);
|
|
6055
|
+
this._windows.set(trackId, window);
|
|
6056
|
+
}
|
|
6057
|
+
return window;
|
|
6058
|
+
}
|
|
6059
|
+
};
|
|
6060
|
+
|
|
6061
|
+
// src/validators/RemoteTrackResolverValidator.ts
|
|
6062
|
+
var UNRESOLVED_TRACK_LINKS_ISSUE = "REMOTE_TRACK_LINKS_UNRESOLVED";
|
|
6063
|
+
var RemoteTrackResolverValidator = class _RemoteTrackResolverValidator {
|
|
6064
|
+
constructor(_observer, onDone, config = {}) {
|
|
6065
|
+
this._observer = _observer;
|
|
6066
|
+
this.onDone = onDone;
|
|
6067
|
+
this._config = {
|
|
6068
|
+
minClients: 2,
|
|
6069
|
+
minInboundTracks: 2,
|
|
6070
|
+
linkedRatioThreshold: 0.5,
|
|
6071
|
+
minChecks: 3,
|
|
6072
|
+
...config
|
|
6073
|
+
};
|
|
6074
|
+
}
|
|
6075
|
+
_observer;
|
|
6076
|
+
onDone;
|
|
6077
|
+
static NAME = "remote-track-resolver";
|
|
6078
|
+
name = _RemoteTrackResolverValidator.NAME;
|
|
6079
|
+
startedAt = Date.now();
|
|
6080
|
+
report = { ready: false };
|
|
6081
|
+
_config;
|
|
6082
|
+
/** Accumulated across every eligible call seen, so one small call cannot decide alone. */
|
|
6083
|
+
_eligibleCalls = 0;
|
|
6084
|
+
_inboundTracks = 0;
|
|
6085
|
+
_linkedInboundTracks = 0;
|
|
6086
|
+
_exampleCallId;
|
|
6087
|
+
_checks = 0;
|
|
6088
|
+
_done = false;
|
|
6089
|
+
/** How many eligible calls were actually examined. `0` means nothing was established. */
|
|
6090
|
+
get checks() {
|
|
6091
|
+
return this._checks;
|
|
6092
|
+
}
|
|
6093
|
+
cancel(reason = "cancelled") {
|
|
6094
|
+
this._finish({ verdict: "inconclusive", reason });
|
|
6095
|
+
}
|
|
6096
|
+
update() {
|
|
6097
|
+
if (this._done) return;
|
|
6098
|
+
for (const call of this._observer.observedCalls.values()) {
|
|
6099
|
+
if (!call.remoteTrackResolver) continue;
|
|
6100
|
+
if (call.observedClients.size < this._config.minClients) continue;
|
|
6101
|
+
let inboundTracks = 0;
|
|
6102
|
+
let linked = 0;
|
|
6103
|
+
for (const client of call.observedClients.values()) {
|
|
6104
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
6105
|
+
for (const inboundTrack of peerConnection.observedInboundTracks.values()) {
|
|
6106
|
+
++inboundTracks;
|
|
6107
|
+
if (inboundTrack.remoteOutboundTrack !== void 0) ++linked;
|
|
6108
|
+
}
|
|
6109
|
+
}
|
|
6110
|
+
}
|
|
6111
|
+
if (inboundTracks < this._config.minInboundTracks) continue;
|
|
6112
|
+
++this._checks;
|
|
6113
|
+
this._eligibleCalls += 1;
|
|
6114
|
+
this._inboundTracks += inboundTracks;
|
|
6115
|
+
this._linkedInboundTracks += linked;
|
|
6116
|
+
this._exampleCallId ??= call.callId;
|
|
6117
|
+
if (0 < this._linkedInboundTracks) {
|
|
6118
|
+
return this._finish({ verdict: "links-resolved", evidence: this._evidence() });
|
|
6119
|
+
}
|
|
6120
|
+
}
|
|
6121
|
+
if (this._config.minChecks <= this._checks && this._linkedInboundTracks === 0) {
|
|
6122
|
+
this._finish({ verdict: "no-links-resolved", evidence: this._evidence() });
|
|
6123
|
+
}
|
|
6124
|
+
}
|
|
6125
|
+
_evidence() {
|
|
6126
|
+
return {
|
|
6127
|
+
eligibleCalls: this._eligibleCalls,
|
|
6128
|
+
inboundTracks: this._inboundTracks,
|
|
6129
|
+
linkedInboundTracks: this._linkedInboundTracks,
|
|
6130
|
+
linkedRatio: 0 < this._inboundTracks ? this._linkedInboundTracks / this._inboundTracks : 0,
|
|
6131
|
+
exampleCallId: this._exampleCallId
|
|
6132
|
+
};
|
|
6133
|
+
}
|
|
6134
|
+
_finish(outcome) {
|
|
6135
|
+
if (this._done) return;
|
|
6136
|
+
this._done = true;
|
|
6137
|
+
const decidedAt = Date.now();
|
|
6138
|
+
this.report = {
|
|
6139
|
+
ready: true,
|
|
6140
|
+
decidedAt,
|
|
6141
|
+
startedAt: this.startedAt,
|
|
6142
|
+
checks: this._checks,
|
|
6143
|
+
...outcome
|
|
6144
|
+
};
|
|
6145
|
+
if (outcome.verdict === "no-links-resolved") {
|
|
6146
|
+
this._observer.addIssue({
|
|
6147
|
+
type: UNRESOLVED_TRACK_LINKS_ISSUE,
|
|
6148
|
+
timestamp: decidedAt,
|
|
6149
|
+
payload: {
|
|
6150
|
+
type: UNRESOLVED_TRACK_LINKS_ISSUE,
|
|
6151
|
+
checks: this._checks,
|
|
6152
|
+
...outcome.evidence,
|
|
6153
|
+
conclusion: {
|
|
6154
|
+
faultDomain: "infrastructure",
|
|
6155
|
+
summary: "a RemoteTrackResolver is configured but has never linked a subscriber to a publisher",
|
|
6156
|
+
recommendation: "check the id the resolver joins on (mediasoup producerId, or your own convention) \u2014 until it links, IssueFanOutDetector, TrackDeliveryMismatchDetector, UnconsumedTrackDetector and SimulcastReceiverValidator all silently do nothing",
|
|
6157
|
+
confidence: 0.9
|
|
6158
|
+
}
|
|
6159
|
+
}
|
|
6160
|
+
});
|
|
6161
|
+
}
|
|
6162
|
+
this.onDone(this.report);
|
|
6163
|
+
}
|
|
6164
|
+
};
|
|
6165
|
+
|
|
6166
|
+
// src/validators/CodecConsistencyValidator.ts
|
|
6167
|
+
var CODEC_MISMATCH_ISSUE = "CODEC_INCONSISTENCY";
|
|
6168
|
+
var CodecConsistencyValidator = class _CodecConsistencyValidator {
|
|
6169
|
+
constructor(_observer, onDone, config = {}) {
|
|
6170
|
+
this._observer = _observer;
|
|
6171
|
+
this.onDone = onDone;
|
|
6172
|
+
this._config = {
|
|
6173
|
+
kinds: ["audio", "video"],
|
|
6174
|
+
minClients: 3,
|
|
6175
|
+
minChecks: 3,
|
|
6176
|
+
...config
|
|
6177
|
+
};
|
|
6178
|
+
}
|
|
6179
|
+
_observer;
|
|
6180
|
+
onDone;
|
|
6181
|
+
static NAME = "codec-consistency";
|
|
6182
|
+
name = _CodecConsistencyValidator.NAME;
|
|
6183
|
+
startedAt = Date.now();
|
|
6184
|
+
report = { ready: false };
|
|
6185
|
+
_config;
|
|
6186
|
+
_inspectedCallIds = /* @__PURE__ */ new Set();
|
|
6187
|
+
_evidence = [];
|
|
6188
|
+
_checks = 0;
|
|
6189
|
+
_done = false;
|
|
6190
|
+
get checks() {
|
|
6191
|
+
return this._checks;
|
|
6192
|
+
}
|
|
6193
|
+
cancel(reason = "cancelled") {
|
|
6194
|
+
this._finish({ verdict: "inconclusive", reason });
|
|
6195
|
+
}
|
|
6196
|
+
update() {
|
|
6197
|
+
if (this._done) return;
|
|
6198
|
+
for (const call of this._observer.observedCalls.values()) {
|
|
6199
|
+
if (this._inspectedCallIds.has(call.callId)) continue;
|
|
6200
|
+
if (call.observedClients.size < this._config.minClients) continue;
|
|
6201
|
+
const tallies = this._tallyOf(call.callId);
|
|
6202
|
+
if (tallies.length === 0) continue;
|
|
6203
|
+
this._inspectedCallIds.add(call.callId);
|
|
6204
|
+
++this._checks;
|
|
6205
|
+
this._evidence.push(...tallies);
|
|
6206
|
+
const split = tallies.find((evidence) => 1 < evidence.mimeTypes.length);
|
|
6207
|
+
if (split) return this._finish({ verdict: "codec-split", evidence: this._evidence });
|
|
6208
|
+
}
|
|
6209
|
+
if (this._checks < this._config.minChecks) return;
|
|
6210
|
+
const unexpected = this._evidence.filter(
|
|
6211
|
+
(evidence) => evidence.expected !== void 0 && evidence.mimeTypes[0] !== evidence.expected
|
|
6212
|
+
);
|
|
6213
|
+
if (0 < unexpected.length) {
|
|
6214
|
+
return this._finish({ verdict: "unexpected-codec", evidence: this._evidence });
|
|
6215
|
+
}
|
|
6216
|
+
this._finish({ verdict: "codec-consistent", evidence: this._evidence });
|
|
6217
|
+
}
|
|
6218
|
+
/** One evidence entry per configured kind that the call actually carried. */
|
|
6219
|
+
_tallyOf(callId) {
|
|
6220
|
+
const call = this._observer.observedCalls.get(callId);
|
|
6221
|
+
if (!call) return [];
|
|
6222
|
+
const byKind = /* @__PURE__ */ new Map();
|
|
6223
|
+
for (const client of call.observedClients.values()) {
|
|
6224
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
6225
|
+
for (const codec of peerConnection.observedCodecs.values()) {
|
|
6226
|
+
const kind = mediaKindOf(codec.mimeType);
|
|
6227
|
+
if (kind === void 0 || !this._config.kinds.includes(kind)) continue;
|
|
6228
|
+
const tally = byKind.get(kind) ?? { mimeTypes: /* @__PURE__ */ new Map() };
|
|
6229
|
+
const clientIds = tally.mimeTypes.get(codec.mimeType) ?? /* @__PURE__ */ new Set();
|
|
6230
|
+
clientIds.add(client.clientId);
|
|
6231
|
+
tally.mimeTypes.set(codec.mimeType, clientIds);
|
|
6232
|
+
byKind.set(kind, tally);
|
|
6233
|
+
}
|
|
6234
|
+
}
|
|
6235
|
+
}
|
|
6236
|
+
const evidence = [];
|
|
6237
|
+
for (const [kind, tally] of byKind) {
|
|
6238
|
+
const ranked = [...tally.mimeTypes.entries()].sort((a, b) => b[1].size - a[1].size);
|
|
6239
|
+
const clients = /* @__PURE__ */ new Set();
|
|
6240
|
+
for (const [, clientIds] of ranked) {
|
|
6241
|
+
for (const clientId of clientIds) clients.add(clientId);
|
|
6242
|
+
}
|
|
6243
|
+
if (clients.size < this._config.minClients) continue;
|
|
6244
|
+
evidence.push({
|
|
6245
|
+
callId,
|
|
6246
|
+
kind,
|
|
6247
|
+
mimeTypes: ranked.map(([mimeType]) => mimeType),
|
|
6248
|
+
clientsPerMimeType: ranked.map(([, clientIds]) => clientIds.size),
|
|
6249
|
+
clients: clients.size,
|
|
6250
|
+
expected: this._config.expected?.[kind]
|
|
6251
|
+
});
|
|
6252
|
+
}
|
|
6253
|
+
return evidence;
|
|
6254
|
+
}
|
|
6255
|
+
_finish(outcome) {
|
|
6256
|
+
if (this._done) return;
|
|
6257
|
+
this._done = true;
|
|
6258
|
+
const decidedAt = Date.now();
|
|
6259
|
+
this.report = {
|
|
6260
|
+
ready: true,
|
|
6261
|
+
decidedAt,
|
|
6262
|
+
startedAt: this.startedAt,
|
|
6263
|
+
checks: this._checks,
|
|
6264
|
+
...outcome
|
|
6265
|
+
};
|
|
6266
|
+
if (outcome.verdict === "codec-split" || outcome.verdict === "unexpected-codec") {
|
|
6267
|
+
this._observer.addIssue({
|
|
6268
|
+
type: CODEC_MISMATCH_ISSUE,
|
|
6269
|
+
timestamp: decidedAt,
|
|
6270
|
+
payload: {
|
|
6271
|
+
type: CODEC_MISMATCH_ISSUE,
|
|
6272
|
+
verdict: outcome.verdict,
|
|
6273
|
+
checks: this._checks,
|
|
6274
|
+
evidence: outcome.evidence,
|
|
6275
|
+
conclusion: {
|
|
6276
|
+
faultDomain: "infrastructure",
|
|
6277
|
+
summary: outcome.verdict === "codec-split" ? "participants of one call are using different codecs \u2014 an SFU that forwards without transcoding cannot serve all of them" : "the deployment is consistently negotiating a codec other than the expected one",
|
|
6278
|
+
recommendation: outcome.verdict === "codec-split" ? "check codec preferences and any SDP munging; a split usually means one endpoint could not negotiate the preferred codec and the others were not renegotiated with it" : "check codec preferences and endpoint support \u2014 a silent fallback keeps working, at a higher bitrate than you budgeted for",
|
|
6279
|
+
confidence: 0.85
|
|
6280
|
+
}
|
|
6281
|
+
}
|
|
6282
|
+
});
|
|
6283
|
+
}
|
|
6284
|
+
this.onDone(this.report);
|
|
6285
|
+
}
|
|
6286
|
+
};
|
|
6287
|
+
function mediaKindOf(mimeType) {
|
|
6288
|
+
if (mimeType.startsWith("video/")) return "video";
|
|
6289
|
+
if (mimeType.startsWith("audio/")) return "audio";
|
|
6290
|
+
return void 0;
|
|
6291
|
+
}
|
|
6292
|
+
|
|
6293
|
+
// src/Observer.ts
|
|
6294
|
+
var logger8 = createLogger("Observer");
|
|
6295
|
+
var Observer = class extends import_events6.EventEmitter {
|
|
6296
|
+
observedTURN = new ObservedTURN();
|
|
3644
6297
|
observedCalls = /* @__PURE__ */ new Map();
|
|
3645
6298
|
observedMediasoupRouters = /* @__PURE__ */ new Map();
|
|
3646
|
-
updater;
|
|
3647
6299
|
/** Ancestry base shared by all Observer-bus events originating at the observer. */
|
|
3648
6300
|
eventScope = { observer: this };
|
|
6301
|
+
config;
|
|
3649
6302
|
closed = false;
|
|
3650
6303
|
totalAddedCall = 0;
|
|
3651
6304
|
totalRemovedCall = 0;
|
|
@@ -3657,33 +6310,179 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
3657
6310
|
numberOfPeerConnections = 0;
|
|
3658
6311
|
/** Global, pre-dispatch middleware chain run on every accepted sample. */
|
|
3659
6312
|
acceptMiddlewares = new MiddlewareProcessor();
|
|
6313
|
+
/**
|
|
6314
|
+
* Fleet-wide index of every open client issue, maintained incrementally as issues open and close.
|
|
6315
|
+
* Each call's index propagates into this one, so cross-call queries cost O(matching issues) rather
|
|
6316
|
+
* than a walk over every call and client. This is what observer-scoped detectors read.
|
|
6317
|
+
*/
|
|
6318
|
+
activeIssuesRegistry = new ActiveIssuesRegistry();
|
|
6319
|
+
/**
|
|
6320
|
+
* Validators currently running. Each removes itself when it finishes, so this is normally empty —
|
|
6321
|
+
* a validator is a one-shot check, not a permanent fixture. Start one with {@link addValidator}.
|
|
6322
|
+
*/
|
|
6323
|
+
validators = /* @__PURE__ */ new Set();
|
|
6324
|
+
/**
|
|
6325
|
+
* Observer-scoped detector registry, run on every `observer.update()` — the place for findings
|
|
6326
|
+
* that span **calls**, e.g. "many calls on the same SFU degraded at once". Detectors raise
|
|
6327
|
+
* findings with `observer.addIssue(...)`, surfaced on the bus as `observer-issue`.
|
|
6328
|
+
* (For findings within a single call use `observedCall.detectors`.)
|
|
6329
|
+
*
|
|
6330
|
+
* Starts **empty**. Populate it with {@link addObserverDetector}, or `detectors.add(...)` for an
|
|
6331
|
+
* instance you built yourself.
|
|
6332
|
+
*/
|
|
6333
|
+
detectors = new Detectors();
|
|
6334
|
+
/**
|
|
6335
|
+
* The call-scoped detectors to build on **every** call this observer creates, in registration
|
|
6336
|
+
* order. Written by {@link addCallDetector}; read by `createObservedCall`.
|
|
6337
|
+
*
|
|
6338
|
+
* Nothing is created implicitly. There is no detector configuration in `ObserverConfig` and no
|
|
6339
|
+
* default set, because a detector that nobody asked for is a detector nobody will act on: it costs
|
|
6340
|
+
* time on every tick and raises findings into a handler that was not written to expect them. An
|
|
6341
|
+
* application says what it wants to watch, or it watches nothing.
|
|
6342
|
+
*
|
|
6343
|
+
* ```ts
|
|
6344
|
+
* observer.addCallDetector('call-concurrent-issue-detector', {
|
|
6345
|
+
* issueTypes: [ 'congestion', 'ice-disconnected' ],
|
|
6346
|
+
* });
|
|
6347
|
+
* ```
|
|
6348
|
+
*/
|
|
6349
|
+
callDetectorConfigs = /* @__PURE__ */ new Map();
|
|
6350
|
+
constructor(config = {}) {
|
|
6351
|
+
super();
|
|
6352
|
+
this.setMaxListeners(Infinity);
|
|
6353
|
+
this.update = this.update.bind(this);
|
|
6354
|
+
this.config = {
|
|
6355
|
+
autoUpdateOnCallUpdate: true,
|
|
6356
|
+
appData: {},
|
|
6357
|
+
closeCallIfEmptyForMs: 6e4,
|
|
6358
|
+
closeClientIfIdleForMs: 6e4,
|
|
6359
|
+
...config
|
|
6360
|
+
};
|
|
6361
|
+
}
|
|
3660
6362
|
get numberOfCalls() {
|
|
3661
6363
|
return this.observedCalls.size;
|
|
3662
6364
|
}
|
|
3663
6365
|
get appData() {
|
|
3664
6366
|
return this.config.appData;
|
|
3665
6367
|
}
|
|
6368
|
+
addObserverDetector(name, config = {}) {
|
|
6369
|
+
if (this.closed) return this;
|
|
6370
|
+
let detector;
|
|
6371
|
+
switch (name) {
|
|
6372
|
+
case SfuCongestionDetector.NAME: {
|
|
6373
|
+
detector = new SfuCongestionDetector(this, config);
|
|
6374
|
+
break;
|
|
6375
|
+
}
|
|
6376
|
+
case ObserverConcurrentIssueDetector.NAME: {
|
|
6377
|
+
detector = new ObserverConcurrentIssueDetector(this, config);
|
|
6378
|
+
break;
|
|
6379
|
+
}
|
|
6380
|
+
case ClientPopulationIssueDetector.NAME: {
|
|
6381
|
+
detector = new ClientPopulationIssueDetector(this, config);
|
|
6382
|
+
break;
|
|
6383
|
+
}
|
|
6384
|
+
case TurnServerHealthDetector.NAME: {
|
|
6385
|
+
detector = new TurnServerHealthDetector(this, config);
|
|
6386
|
+
break;
|
|
6387
|
+
}
|
|
6388
|
+
case TurnServerOutageDetector.NAME: {
|
|
6389
|
+
detector = new TurnServerOutageDetector(this, config);
|
|
6390
|
+
break;
|
|
6391
|
+
}
|
|
6392
|
+
default: {
|
|
6393
|
+
logger8.warn("Unknown detector name %s; skipping", name);
|
|
6394
|
+
return this;
|
|
6395
|
+
}
|
|
6396
|
+
}
|
|
6397
|
+
this.detectors.add(detector);
|
|
6398
|
+
return this;
|
|
6399
|
+
}
|
|
6400
|
+
/**
|
|
6401
|
+
* Enable a call-scoped detector for calls created **from now on**.
|
|
6402
|
+
*
|
|
6403
|
+
* This edits the config, not the live calls: calls already open keep the detector set they were
|
|
6404
|
+
* built with. To add one to an existing call, use `observedCall.addDetector(...)` directly.
|
|
6405
|
+
*/
|
|
6406
|
+
addCallDetector(name, config = {}) {
|
|
6407
|
+
if (this.closed) return this;
|
|
6408
|
+
this.callDetectorConfigs.set(name, config);
|
|
6409
|
+
return this;
|
|
6410
|
+
}
|
|
6411
|
+
/** Stop building `name` on calls created from now on. Calls already open are untouched. */
|
|
6412
|
+
removeCallDetector(name) {
|
|
6413
|
+
this.callDetectorConfigs.delete(name);
|
|
6414
|
+
return this;
|
|
6415
|
+
}
|
|
6416
|
+
/**
|
|
6417
|
+
* Start a structural check. It runs on each `observer.update()` until it can decide, reports once
|
|
6418
|
+
* on `validation-ready`, and removes itself.
|
|
6419
|
+
*
|
|
6420
|
+
* ```ts
|
|
6421
|
+
* observer.validate('simulcast-receiver-validator', { minChecks: 5 });
|
|
6422
|
+
* ```
|
|
6423
|
+
*
|
|
6424
|
+
* Config keys are optional and merged over that validator's defaults. Call it again — after a
|
|
6425
|
+
* deploy, say — to check again; there is no revalidation timer, because a deploy rather than
|
|
6426
|
+
* elapsed time is what makes a structural verdict stale.
|
|
6427
|
+
*/
|
|
6428
|
+
addValidator(name, config = {}) {
|
|
6429
|
+
if (this.closed) return this;
|
|
6430
|
+
let validator;
|
|
6431
|
+
const onDone = (report) => {
|
|
6432
|
+
if (validator) this.validators.delete(validator);
|
|
6433
|
+
this._notify("validation-ready", { ...this.eventScope, report, validator: name });
|
|
6434
|
+
};
|
|
6435
|
+
switch (name) {
|
|
6436
|
+
case SimulcastReceiverValidator.NAME:
|
|
6437
|
+
validator = new SimulcastReceiverValidator(this, onDone, config);
|
|
6438
|
+
break;
|
|
6439
|
+
case RemoteTrackResolverValidator.NAME:
|
|
6440
|
+
validator = new RemoteTrackResolverValidator(this, onDone, config);
|
|
6441
|
+
break;
|
|
6442
|
+
case CodecConsistencyValidator.NAME:
|
|
6443
|
+
validator = new CodecConsistencyValidator(this, onDone, config);
|
|
6444
|
+
break;
|
|
6445
|
+
}
|
|
6446
|
+
if (!validator) {
|
|
6447
|
+
logger8.warn("Unknown validator name %s; skipping", name);
|
|
6448
|
+
return this;
|
|
6449
|
+
}
|
|
6450
|
+
this.validators.add(validator);
|
|
6451
|
+
return this;
|
|
6452
|
+
}
|
|
3666
6453
|
getObservedCall(callId) {
|
|
3667
6454
|
if (this.closed || !this.observedCalls.has(callId)) return;
|
|
3668
6455
|
return this.observedCalls.get(callId);
|
|
3669
6456
|
}
|
|
3670
6457
|
createObservedCall(settings) {
|
|
3671
6458
|
if (this.closed) {
|
|
3672
|
-
|
|
6459
|
+
logger8.warn("Attempted to create a call (callId: %s) on a closed observer", settings.callId);
|
|
3673
6460
|
return void 0;
|
|
3674
6461
|
}
|
|
3675
6462
|
if (this.observedCalls.has(settings.callId)) {
|
|
3676
|
-
|
|
6463
|
+
logger8.warn("Observed Call with id %s already exists; returning the existing instance", settings.callId);
|
|
3677
6464
|
return this.observedCalls.get(settings.callId);
|
|
3678
6465
|
}
|
|
3679
6466
|
const callSettings = {
|
|
3680
6467
|
...settings,
|
|
3681
|
-
updatePolicy: settings.updatePolicy ?? this.config.defaultCallUpdatePolicy,
|
|
3682
6468
|
closeCallIfEmptyForMs: settings.closeCallIfEmptyForMs ?? this.config.closeCallIfEmptyForMs,
|
|
3683
6469
|
appData: settings.appData ?? this.config.createCallAppData?.({ callId: settings.callId, observer: this })
|
|
3684
6470
|
};
|
|
3685
|
-
const
|
|
6471
|
+
const callActiveIssuesRegistry = new ActiveIssuesRegistry(this.activeIssuesRegistry);
|
|
6472
|
+
const observedCall = new ObservedCall(
|
|
6473
|
+
callSettings,
|
|
6474
|
+
this,
|
|
6475
|
+
callActiveIssuesRegistry
|
|
6476
|
+
);
|
|
3686
6477
|
observedCall.remoteTrackResolver = this.config.createRemoteTrackResolver?.(observedCall);
|
|
6478
|
+
for (const [name, detectorConfig] of this.callDetectorConfigs) {
|
|
6479
|
+
observedCall.addDetector(name, detectorConfig);
|
|
6480
|
+
}
|
|
6481
|
+
if (this.config.autoUpdateOnCallUpdate) {
|
|
6482
|
+
const onCallUpdate = () => this.update();
|
|
6483
|
+
observedCall.once("close", () => observedCall.off("update", onCallUpdate));
|
|
6484
|
+
observedCall.on("update", onCallUpdate);
|
|
6485
|
+
}
|
|
3687
6486
|
observedCall.once("close", () => {
|
|
3688
6487
|
this.observedCalls.delete(observedCall.callId);
|
|
3689
6488
|
++this.totalRemovedCall;
|
|
@@ -3698,11 +6497,11 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
3698
6497
|
}
|
|
3699
6498
|
createObservedMediasoupRouter(settings) {
|
|
3700
6499
|
if (this.closed) {
|
|
3701
|
-
|
|
6500
|
+
logger8.warn("Attempted to create mediasoup router (id: %d) on a closed observer", settings.router.id);
|
|
3702
6501
|
return void 0;
|
|
3703
6502
|
}
|
|
3704
6503
|
if (this.observedMediasoupRouters.has(settings.router.id)) {
|
|
3705
|
-
|
|
6504
|
+
logger8.warn("Observed Mediasoup Router (id %s) already exists; returning the existing instance", settings.router.id);
|
|
3706
6505
|
return this.observedMediasoupRouters.get(settings.router.id);
|
|
3707
6506
|
}
|
|
3708
6507
|
const observedMediasoupRouter = new ObservedMediasoupRouter(settings);
|
|
@@ -3733,10 +6532,14 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
3733
6532
|
}
|
|
3734
6533
|
close() {
|
|
3735
6534
|
if (this.closed) {
|
|
3736
|
-
return
|
|
6535
|
+
return logger8.debug("Attempted to close twice");
|
|
3737
6536
|
}
|
|
3738
6537
|
this.closed = true;
|
|
3739
|
-
this.observedCalls.
|
|
6538
|
+
for (const call of [...this.observedCalls.values()]) call.close();
|
|
6539
|
+
this.detectors.clear();
|
|
6540
|
+
for (const validator of [...this.validators]) validator.cancel();
|
|
6541
|
+
this.validators.clear();
|
|
6542
|
+
this.activeIssuesRegistry.clear();
|
|
3740
6543
|
this._notify("observer-closed", { ...this.eventScope });
|
|
3741
6544
|
}
|
|
3742
6545
|
accept(sample, context) {
|
|
@@ -3746,7 +6549,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
3746
6549
|
try {
|
|
3747
6550
|
this.acceptMiddlewares.process({ sample, context });
|
|
3748
6551
|
} catch (err) {
|
|
3749
|
-
|
|
6552
|
+
logger8.warn("An accept middleware threw; dropping the sample. %o", err);
|
|
3750
6553
|
}
|
|
3751
6554
|
if (!sample.callId) {
|
|
3752
6555
|
this._notify("sample-rejected", { ...this.eventScope, reason: "missing-callId", sample });
|
|
@@ -3793,14 +6596,251 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
3793
6596
|
this.numberOfClientsUsingTurn += call.clientsUsedTurn.size;
|
|
3794
6597
|
}
|
|
3795
6598
|
this.observedTURN.update();
|
|
6599
|
+
this.detectors.update();
|
|
6600
|
+
for (const validator of [...this.validators]) {
|
|
6601
|
+
try {
|
|
6602
|
+
validator.update();
|
|
6603
|
+
} catch (err) {
|
|
6604
|
+
logger8.warn("Error running validator %s: %o", validator.name, err);
|
|
6605
|
+
}
|
|
6606
|
+
}
|
|
3796
6607
|
this._notify("observer-updated", { ...this.eventScope });
|
|
3797
6608
|
}
|
|
6609
|
+
/**
|
|
6610
|
+
* Raise an observer-scoped (cross-call / SFU-wide) finding. Emitted on the bus as
|
|
6611
|
+
* `observer-issue`. Intended for `observer.detectors`, but the application may call it too.
|
|
6612
|
+
*
|
|
6613
|
+
* `payload` takes an **object**; see `ObserverIssue`.
|
|
6614
|
+
*/
|
|
6615
|
+
addIssue(issue) {
|
|
6616
|
+
if (this.closed) return;
|
|
6617
|
+
this._notify("observer-issue", { ...this.eventScope, issue });
|
|
6618
|
+
}
|
|
3798
6619
|
/** Emit an Observer-bus event. */
|
|
3799
6620
|
_notify(type, ...args) {
|
|
3800
6621
|
this.emit(type, ...args);
|
|
3801
6622
|
}
|
|
3802
6623
|
};
|
|
3803
6624
|
|
|
6625
|
+
// src/utils/CallHealthAggregator.ts
|
|
6626
|
+
var defaultClientHealthThresholds = {
|
|
6627
|
+
inboundFractionLost: 0.03,
|
|
6628
|
+
outboundFractionLost: 0.03,
|
|
6629
|
+
rttInMs: 400,
|
|
6630
|
+
freezeCount: 1,
|
|
6631
|
+
concealmentRatio: 0.1
|
|
6632
|
+
};
|
|
6633
|
+
var CallHealthAggregator = class {
|
|
6634
|
+
constructor(_call, thresholds = defaultClientHealthThresholds) {
|
|
6635
|
+
this._call = _call;
|
|
6636
|
+
this.thresholds = thresholds;
|
|
6637
|
+
}
|
|
6638
|
+
_call;
|
|
6639
|
+
thresholds;
|
|
6640
|
+
aggregate() {
|
|
6641
|
+
const clients = [];
|
|
6642
|
+
for (const client of this._call.observedClients.values()) {
|
|
6643
|
+
clients.push(this._clientHealth(client));
|
|
6644
|
+
}
|
|
6645
|
+
const degraded = clients.filter((c) => c.degraded);
|
|
6646
|
+
const inboundDegraded = clients.filter((c) => c.inboundDegraded);
|
|
6647
|
+
const outboundDegraded = clients.filter((c) => c.outboundDegraded);
|
|
6648
|
+
const collect = (pick) => summarize(clients.map(pick).filter((v) => typeof v === "number" && Number.isFinite(v)));
|
|
6649
|
+
const qualityLimitation = { cpu: 0, bandwidth: 0, other: 0 };
|
|
6650
|
+
for (const client of clients) {
|
|
6651
|
+
if (client.qualityLimitationReasons.includes("cpu")) qualityLimitation.cpu += 1;
|
|
6652
|
+
if (client.qualityLimitationReasons.includes("bandwidth")) qualityLimitation.bandwidth += 1;
|
|
6653
|
+
if (client.qualityLimitationReasons.some((r) => r !== "cpu" && r !== "bandwidth")) qualityLimitation.other += 1;
|
|
6654
|
+
}
|
|
6655
|
+
const ratio = (n) => 0 < clients.length ? n / clients.length : 0;
|
|
6656
|
+
return {
|
|
6657
|
+
callId: this._call.callId,
|
|
6658
|
+
clients,
|
|
6659
|
+
numberOfClients: clients.length,
|
|
6660
|
+
numberOfDegradedClients: degraded.length,
|
|
6661
|
+
numberOfInboundDegradedClients: inboundDegraded.length,
|
|
6662
|
+
numberOfOutboundDegradedClients: outboundDegraded.length,
|
|
6663
|
+
degradedRatio: ratio(degraded.length),
|
|
6664
|
+
inboundDegradedRatio: ratio(inboundDegraded.length),
|
|
6665
|
+
outboundDegradedRatio: ratio(outboundDegraded.length),
|
|
6666
|
+
rttInMs: collect((c) => c.rttInMs),
|
|
6667
|
+
inboundFractionLost: collect((c) => c.inboundFractionLost),
|
|
6668
|
+
concealmentRatio: collect((c) => c.concealmentRatio),
|
|
6669
|
+
qualityLimitation,
|
|
6670
|
+
freezes: {
|
|
6671
|
+
affectedClients: clients.filter((c) => 0 < c.deltaFreezeCount).length,
|
|
6672
|
+
total: clients.reduce((sum, c) => sum + c.deltaFreezeCount, 0)
|
|
6673
|
+
}
|
|
6674
|
+
};
|
|
6675
|
+
}
|
|
6676
|
+
_clientHealth(client) {
|
|
6677
|
+
let deltaFreezeCount = 0;
|
|
6678
|
+
let concealedSamples = 0;
|
|
6679
|
+
let receivedSamples = 0;
|
|
6680
|
+
let remoteLostPackets = 0;
|
|
6681
|
+
let remoteSentPackets = 0;
|
|
6682
|
+
const qualityLimitationReasons = /* @__PURE__ */ new Set();
|
|
6683
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
6684
|
+
for (const rtp of peerConnection.observedInboundRtps.values()) {
|
|
6685
|
+
deltaFreezeCount += rtp.deltaFreezeCount;
|
|
6686
|
+
concealedSamples += rtp.deltaConcealedSamples;
|
|
6687
|
+
receivedSamples += rtp.deltaReceivedSamples;
|
|
6688
|
+
}
|
|
6689
|
+
for (const rtp of peerConnection.observedOutboundRtps.values()) {
|
|
6690
|
+
if (rtp.qualityLimitationReason && rtp.qualityLimitationReason !== "none") {
|
|
6691
|
+
qualityLimitationReasons.add(rtp.qualityLimitationReason);
|
|
6692
|
+
}
|
|
6693
|
+
if (rtp.remoteFractionLost !== void 0) {
|
|
6694
|
+
remoteLostPackets += rtp.remoteFractionLost * rtp.deltaPacketsSent;
|
|
6695
|
+
remoteSentPackets += rtp.deltaPacketsSent;
|
|
6696
|
+
}
|
|
6697
|
+
}
|
|
6698
|
+
}
|
|
6699
|
+
const inboundTotal = client.deltaInboundPacketsLost + client.deltaInboundPacketsReceived;
|
|
6700
|
+
const inboundFractionLost = 0 < inboundTotal ? client.deltaInboundPacketsLost / inboundTotal : void 0;
|
|
6701
|
+
const outboundFractionLost = 0 < remoteSentPackets ? remoteLostPackets / remoteSentPackets : void 0;
|
|
6702
|
+
const concealmentRatio = 0 < receivedSamples ? concealedSamples / receivedSamples : void 0;
|
|
6703
|
+
const rttInMs = client.currentAvgRttInMs;
|
|
6704
|
+
const reasons = [];
|
|
6705
|
+
if (this.thresholds.inboundFractionLost < (inboundFractionLost ?? 0)) reasons.push("inbound-fraction-lost");
|
|
6706
|
+
if (this.thresholds.freezeCount <= deltaFreezeCount) reasons.push("freezes");
|
|
6707
|
+
if (this.thresholds.concealmentRatio < (concealmentRatio ?? 0)) reasons.push("concealment");
|
|
6708
|
+
if (this.thresholds.rttInMs < (rttInMs ?? 0)) reasons.push("rtt");
|
|
6709
|
+
const outboundReasons = [];
|
|
6710
|
+
if (this.thresholds.outboundFractionLost < (outboundFractionLost ?? 0)) outboundReasons.push("outbound-fraction-lost");
|
|
6711
|
+
if (0 < qualityLimitationReasons.size) {
|
|
6712
|
+
outboundReasons.push(...[...qualityLimitationReasons].map((r) => `quality-limited-${r}`));
|
|
6713
|
+
}
|
|
6714
|
+
const inboundDegraded = 0 < reasons.length;
|
|
6715
|
+
const outboundDegraded = 0 < outboundReasons.length;
|
|
6716
|
+
return {
|
|
6717
|
+
observedClient: client,
|
|
6718
|
+
clientId: client.clientId,
|
|
6719
|
+
inboundDegraded,
|
|
6720
|
+
outboundDegraded,
|
|
6721
|
+
degraded: inboundDegraded || outboundDegraded,
|
|
6722
|
+
reasons: [...reasons, ...outboundReasons],
|
|
6723
|
+
inboundFractionLost,
|
|
6724
|
+
outboundFractionLost,
|
|
6725
|
+
rttInMs,
|
|
6726
|
+
deltaFreezeCount,
|
|
6727
|
+
concealmentRatio,
|
|
6728
|
+
qualityLimitationReasons: [...qualityLimitationReasons],
|
|
6729
|
+
usingTURN: client.usingTURN,
|
|
6730
|
+
usingTCP: client.usingTCP
|
|
6731
|
+
};
|
|
6732
|
+
}
|
|
6733
|
+
};
|
|
6734
|
+
|
|
6735
|
+
// src/common/ObserverIssue.ts
|
|
6736
|
+
function issuePayloadOf(issue) {
|
|
6737
|
+
const payload = issue.payload;
|
|
6738
|
+
if (payload === void 0) return void 0;
|
|
6739
|
+
return typeof payload === "string" ? parseJsonObject(payload) : payload;
|
|
6740
|
+
}
|
|
6741
|
+
function issuePayloadAsString(issue) {
|
|
6742
|
+
const payload = issue.payload;
|
|
6743
|
+
if (payload === void 0) return void 0;
|
|
6744
|
+
if (typeof payload === "string") return payload;
|
|
6745
|
+
try {
|
|
6746
|
+
return JSON.stringify(payload);
|
|
6747
|
+
} catch {
|
|
6748
|
+
return void 0;
|
|
6749
|
+
}
|
|
6750
|
+
}
|
|
6751
|
+
|
|
6752
|
+
// src/utils/TrendTester.ts
|
|
6753
|
+
var TrendTester = class {
|
|
6754
|
+
_size;
|
|
6755
|
+
_values = [];
|
|
6756
|
+
_tieCounts = /* @__PURE__ */ new Map();
|
|
6757
|
+
_pageHinkleyDelta;
|
|
6758
|
+
_pageHinkleyLambda;
|
|
6759
|
+
_mannKendallAlpha;
|
|
6760
|
+
_s = 0;
|
|
6761
|
+
_rejected = 0;
|
|
6762
|
+
_pageHinkleyResult;
|
|
6763
|
+
constructor(config = {}) {
|
|
6764
|
+
this._size = Math.max(2, Math.floor(config.size ?? 30));
|
|
6765
|
+
this._pageHinkleyDelta = config.pageHinkleyDelta ?? 0;
|
|
6766
|
+
this._pageHinkleyLambda = config.pageHinkleyLambda ?? 50;
|
|
6767
|
+
this._mannKendallAlpha = config.mannKendallAlpha ?? 0.05;
|
|
6768
|
+
}
|
|
6769
|
+
/** Values rejected by {@link add} for being non-finite. Non-zero means the caller has a bug. */
|
|
6770
|
+
get rejected() {
|
|
6771
|
+
return this._rejected;
|
|
6772
|
+
}
|
|
6773
|
+
/** How many values are currently in the window (`<= size`). */
|
|
6774
|
+
get length() {
|
|
6775
|
+
return this._values.length;
|
|
6776
|
+
}
|
|
6777
|
+
/** The configured window length. */
|
|
6778
|
+
get size() {
|
|
6779
|
+
return this._size;
|
|
6780
|
+
}
|
|
6781
|
+
/**
|
|
6782
|
+
* Add the next value in the stream, evicting the oldest once the window is full.
|
|
6783
|
+
*
|
|
6784
|
+
* **Non-finite values are rejected** rather than stored, and the rejection is counted in
|
|
6785
|
+
* {@link rejected}. This is not defensive noise — it is the difference between a bad reading and
|
|
6786
|
+
* a bad instance. `Math.sign(NaN)` is `NaN`, so a single `NaN` would poison the incremental
|
|
6787
|
+
* Mann-Kendall sum `_s` **permanently**: every later `add` and `_evictOldest` adds or subtracts
|
|
6788
|
+
* `NaN`, the z-score is `NaN`, every comparison against it is `false`, and the tester silently
|
|
6789
|
+
* reports `no-trend` forever after. It would also take a `NaN` key in `_tieCounts` that can never
|
|
6790
|
+
* be matched on eviction, since `NaN !== NaN`.
|
|
6791
|
+
*
|
|
6792
|
+
* `undefined` RTT (no measurement this tick) must not be coerced to `0` and passed in either —
|
|
6793
|
+
* "we didn't measure" is not "the trip took no time", and feeding zeros manufactures a downward
|
|
6794
|
+
* trend. Skip the sample instead.
|
|
6795
|
+
*/
|
|
6796
|
+
add(value) {
|
|
6797
|
+
if (!Number.isFinite(value)) {
|
|
6798
|
+
++this._rejected;
|
|
6799
|
+
return;
|
|
6800
|
+
}
|
|
6801
|
+
if (this._values.length === this._size) this._evictOldest();
|
|
6802
|
+
for (let i = 0; i < this._values.length; i++) this._s += Math.sign(value - this._values[i]);
|
|
6803
|
+
this._values.push(value);
|
|
6804
|
+
this._bumpTie(value, 1);
|
|
6805
|
+
this._pageHinkleyResult = pageHinkley(this._values, this._pageHinkleyDelta, this._pageHinkleyLambda);
|
|
6806
|
+
}
|
|
6807
|
+
/** The current Page-Hinkley read-out over the window. `undefined` before the first value. */
|
|
6808
|
+
pageHinkley() {
|
|
6809
|
+
return this._pageHinkleyResult;
|
|
6810
|
+
}
|
|
6811
|
+
/** The current Mann-Kendall read-out over the window. */
|
|
6812
|
+
mannKendall() {
|
|
6813
|
+
const n = this._values.length;
|
|
6814
|
+
if (n < 2) return { s: 0, variance: 0, z: 0, pValue: 1, trend: "no-trend" };
|
|
6815
|
+
let tieCorrection = 0;
|
|
6816
|
+
for (const groupSize of this._tieCounts.values()) {
|
|
6817
|
+
if (1 < groupSize) tieCorrection += groupSize * (groupSize - 1) * (2 * groupSize + 5);
|
|
6818
|
+
}
|
|
6819
|
+
const variance = (n * (n - 1) * (2 * n + 5) - tieCorrection) / 18;
|
|
6820
|
+
return { s: this._s, variance, ...mannKendallVerdict(this._s, variance, this._mannKendallAlpha) };
|
|
6821
|
+
}
|
|
6822
|
+
/** Drop everything, e.g. after a detected change point, to start judging the trend fresh. */
|
|
6823
|
+
clear() {
|
|
6824
|
+
this._values.length = 0;
|
|
6825
|
+
this._tieCounts.clear();
|
|
6826
|
+
this._s = 0;
|
|
6827
|
+
this._rejected = 0;
|
|
6828
|
+
this._pageHinkleyResult = void 0;
|
|
6829
|
+
}
|
|
6830
|
+
/** Remove the oldest value from the window and correct `_s` for the pairs it was part of. */
|
|
6831
|
+
_evictOldest() {
|
|
6832
|
+
const removed = this._values.shift();
|
|
6833
|
+
if (removed === void 0) return;
|
|
6834
|
+
for (let i = 0; i < this._values.length; i++) this._s -= Math.sign(this._values[i] - removed);
|
|
6835
|
+
this._bumpTie(removed, -1);
|
|
6836
|
+
}
|
|
6837
|
+
_bumpTie(value, delta) {
|
|
6838
|
+
const count = (this._tieCounts.get(value) ?? 0) + delta;
|
|
6839
|
+
if (count <= 0) this._tieCounts.delete(value);
|
|
6840
|
+
else this._tieCounts.set(value, count);
|
|
6841
|
+
}
|
|
6842
|
+
};
|
|
6843
|
+
|
|
3804
6844
|
// src/sinks/ClientSampleSink.ts
|
|
3805
6845
|
var import_events7 = require("events");
|
|
3806
6846
|
var ClientSampleSink = class extends import_events7.EventEmitter {
|
|
@@ -3864,7 +6904,7 @@ function createInMemorySink(samples) {
|
|
|
3864
6904
|
return new InMemorySink(samples);
|
|
3865
6905
|
}
|
|
3866
6906
|
|
|
3867
|
-
// src/
|
|
6907
|
+
// src/resolvers/RemoteTrackResolver.ts
|
|
3868
6908
|
var RemoteTrackResolver = class {
|
|
3869
6909
|
constructor(observedCall, resolvers) {
|
|
3870
6910
|
this.observedCall = observedCall;
|
|
@@ -3923,10 +6963,15 @@ var RemoteTrackResolver = class {
|
|
|
3923
6963
|
if (!outboundTrack) return;
|
|
3924
6964
|
outboundTrack.remoteInboundTracks.add(inboundTrack);
|
|
3925
6965
|
inboundTrack.remoteOutboundTrack = outboundTrack;
|
|
6966
|
+
this.observedCall.unconsumedOutboundTracks.delete(outboundTrack);
|
|
3926
6967
|
}
|
|
3927
6968
|
_removeInboundTrack(inboundTrack) {
|
|
3928
|
-
inboundTrack.remoteOutboundTrack
|
|
6969
|
+
const outboundTrack = inboundTrack.remoteOutboundTrack;
|
|
6970
|
+
outboundTrack?.remoteInboundTracks.delete(inboundTrack);
|
|
3929
6971
|
inboundTrack.remoteOutboundTrack = void 0;
|
|
6972
|
+
if (outboundTrack && outboundTrack.remoteInboundTracks.size === 0) {
|
|
6973
|
+
this.observedCall.unconsumedOutboundTracks.add(outboundTrack);
|
|
6974
|
+
}
|
|
3930
6975
|
const subscriberId = this.resolvers.resolveInboundTrackSubscriberId?.(inboundTrack);
|
|
3931
6976
|
if (subscriberId) {
|
|
3932
6977
|
this._subscriberIdToInboundTrack.delete(subscriberId);
|
|
@@ -3936,12 +6981,14 @@ var RemoteTrackResolver = class {
|
|
|
3936
6981
|
const publisherId = this.resolvers.resolveOutboundTrackPublisherId(outboundTrack);
|
|
3937
6982
|
if (!publisherId) return;
|
|
3938
6983
|
this._publisherIdToOutboundTrack.set(publisherId, outboundTrack);
|
|
6984
|
+
this.observedCall.unconsumedOutboundTracks.add(outboundTrack);
|
|
3939
6985
|
for (const observedClient of this.observedCall.observedClients.values()) {
|
|
3940
6986
|
for (const observedPeerConnection of observedClient.observedPeerConnections.values()) {
|
|
3941
6987
|
for (const inboundTrack of observedPeerConnection.observedInboundTracks.values()) {
|
|
3942
6988
|
if (this.resolvers.resolveInboundTrackPublisherId(inboundTrack) !== publisherId) continue;
|
|
3943
6989
|
outboundTrack.remoteInboundTracks.add(inboundTrack);
|
|
3944
6990
|
inboundTrack.remoteOutboundTrack = outboundTrack;
|
|
6991
|
+
this.observedCall.unconsumedOutboundTracks.delete(outboundTrack);
|
|
3945
6992
|
}
|
|
3946
6993
|
}
|
|
3947
6994
|
}
|
|
@@ -3955,10 +7002,11 @@ var RemoteTrackResolver = class {
|
|
|
3955
7002
|
inboundTrack.remoteOutboundTrack = void 0;
|
|
3956
7003
|
}
|
|
3957
7004
|
outboundTrack.remoteInboundTracks.clear();
|
|
7005
|
+
this.observedCall.unconsumedOutboundTracks.delete(outboundTrack);
|
|
3958
7006
|
}
|
|
3959
7007
|
};
|
|
3960
7008
|
|
|
3961
|
-
// src/
|
|
7009
|
+
// src/resolvers/RemoteTrackResolverFactories.ts
|
|
3962
7010
|
function createDefaultMediasoupRemoteTrackResolverFactory() {
|
|
3963
7011
|
return (observedCall) => new RemoteTrackResolver(observedCall, {
|
|
3964
7012
|
resolveInboundTrackPublisherId: (inboundTrack) => inboundTrack.attachments?.producerId,
|
|
@@ -3985,15 +7033,27 @@ function createP2pRemoteTrackResolverFactory() {
|
|
|
3985
7033
|
}
|
|
3986
7034
|
// Annotate the CommonJS export names for ESM import in node:
|
|
3987
7035
|
0 && (module.exports = {
|
|
7036
|
+
ActiveIssuesRegistry,
|
|
7037
|
+
CODEC_MISMATCH_ISSUE,
|
|
7038
|
+
CallConcurrentIssueDetector,
|
|
7039
|
+
CallConcurrentIssueTypes,
|
|
7040
|
+
CallHealthAggregator,
|
|
3988
7041
|
ClientEventTypes,
|
|
3989
7042
|
ClientMetaTypes,
|
|
7043
|
+
ClientPopulationIssueDetector,
|
|
7044
|
+
ClientPopulationIssueTypes,
|
|
3990
7045
|
ClientSampleSink,
|
|
7046
|
+
CodecConsistencyValidator,
|
|
3991
7047
|
Detectors,
|
|
3992
7048
|
InMemorySink,
|
|
7049
|
+
IssueFanOutDetector,
|
|
7050
|
+
IssueFanOutTypes,
|
|
3993
7051
|
JsonlFileSink,
|
|
7052
|
+
LOWEST_COMMON_DENOMINATOR_ISSUE,
|
|
3994
7053
|
ObservedCall,
|
|
3995
7054
|
ObservedCertificate,
|
|
3996
7055
|
ObservedClient,
|
|
7056
|
+
ObservedClientIssueRegistry,
|
|
3997
7057
|
ObservedCodec,
|
|
3998
7058
|
ObservedDataChannel,
|
|
3999
7059
|
ObservedIceCandidate,
|
|
@@ -4011,13 +7071,50 @@ function createP2pRemoteTrackResolverFactory() {
|
|
|
4011
7071
|
ObservedRemoteInboundRtp,
|
|
4012
7072
|
ObservedRemoteOutboundRtp,
|
|
4013
7073
|
Observer,
|
|
7074
|
+
ObserverConcurrentIssueDetector,
|
|
7075
|
+
ObserverConcurrentIssueTypes,
|
|
7076
|
+
PublisherFaultCorroborationDetector,
|
|
7077
|
+
PublisherFaultTypes,
|
|
7078
|
+
RESOLVED_ISSUE_SUFFIX,
|
|
4014
7079
|
RemoteTrackResolver,
|
|
7080
|
+
RemoteTrackResolverValidator,
|
|
7081
|
+
SfuCongestionDetector,
|
|
7082
|
+
SimulcastReceiverValidator,
|
|
7083
|
+
SlidingWindow,
|
|
7084
|
+
TrackDeliveryMismatchDetector,
|
|
7085
|
+
TrackDeliveryMismatchTypes,
|
|
7086
|
+
TrendTester,
|
|
7087
|
+
TurnServerHealthDetector,
|
|
7088
|
+
TurnServerHealthTypes,
|
|
7089
|
+
TurnServerOutageDetector,
|
|
7090
|
+
TurnServerOutageTypes,
|
|
7091
|
+
UNRESOLVED_TRACK_LINKS_ISSUE,
|
|
7092
|
+
UnconsumedTrackDetector,
|
|
7093
|
+
UnconsumedTrackTypes,
|
|
7094
|
+
baseIssueType,
|
|
7095
|
+
concludeCallIssue,
|
|
7096
|
+
concludeObserverIssue,
|
|
7097
|
+
correlation,
|
|
7098
|
+
counterDelta,
|
|
4015
7099
|
createDefaultMediasoupRemoteTrackResolverFactory,
|
|
4016
7100
|
createInMemorySink,
|
|
4017
7101
|
createJsonlFileSink,
|
|
4018
7102
|
createJsonlFileSinkFactory,
|
|
4019
7103
|
createLogger,
|
|
4020
7104
|
createP2pRemoteTrackResolverFactory,
|
|
4021
|
-
|
|
7105
|
+
defaultClientHealthThresholds,
|
|
7106
|
+
isClientIssueResolutionEntry,
|
|
7107
|
+
issuePayloadAsString,
|
|
7108
|
+
issuePayloadOf,
|
|
7109
|
+
mannKendall,
|
|
7110
|
+
mannKendallVerdict,
|
|
7111
|
+
median,
|
|
7112
|
+
medianAbsoluteDeviation,
|
|
7113
|
+
pageHinkley,
|
|
7114
|
+
percentile,
|
|
7115
|
+
percentileOfSorted,
|
|
7116
|
+
robustZScore,
|
|
7117
|
+
setObserverLogger,
|
|
7118
|
+
summarize
|
|
4022
7119
|
});
|
|
4023
7120
|
//# sourceMappingURL=index.js.map
|