@observertc/observer-js 1.0.0-beta.16 → 1.0.0-beta.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +174 -21
- package/dist/index.d.mts +429 -135
- package/dist/index.d.ts +429 -135
- package/dist/index.js +274 -101
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +271 -100
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -35,6 +35,7 @@ __export(src_exports, {
|
|
|
35
35
|
CallConcurrentIssueDetector: () => CallConcurrentIssueDetector,
|
|
36
36
|
CallConcurrentIssueTypes: () => CallConcurrentIssueTypes,
|
|
37
37
|
CallHealthAggregator: () => CallHealthAggregator,
|
|
38
|
+
CallSummaryCollector: () => CallSummaryCollector,
|
|
38
39
|
ClientEventTypes: () => ClientEventTypes,
|
|
39
40
|
ClientMetaTypes: () => ClientMetaTypes,
|
|
40
41
|
ClientPopulationIssueDetector: () => ClientPopulationIssueDetector,
|
|
@@ -93,16 +94,17 @@ __export(src_exports, {
|
|
|
93
94
|
concludeObserverIssue: () => concludeObserverIssue,
|
|
94
95
|
correlation: () => correlation,
|
|
95
96
|
counterDelta: () => counterDelta,
|
|
97
|
+
createCallSummary: () => createCallSummary,
|
|
96
98
|
createDefaultMediasoupRemoteTrackResolverFactory: () => createDefaultMediasoupRemoteTrackResolverFactory,
|
|
97
99
|
createInMemorySink: () => createInMemorySink,
|
|
98
100
|
createJsonlFileSink: () => createJsonlFileSink,
|
|
99
101
|
createJsonlFileSinkFactory: () => createJsonlFileSinkFactory,
|
|
100
102
|
createLogger: () => createLogger,
|
|
101
103
|
createP2pRemoteTrackResolverFactory: () => createP2pRemoteTrackResolverFactory,
|
|
104
|
+
defaultCallSummaryConfig: () => defaultCallSummaryConfig,
|
|
102
105
|
defaultClientHealthThresholds: () => defaultClientHealthThresholds,
|
|
103
106
|
isClientIssueResolutionEntry: () => isClientIssueResolutionEntry,
|
|
104
107
|
issuePayloadAsString: () => issuePayloadAsString,
|
|
105
|
-
issuePayloadOf: () => issuePayloadOf,
|
|
106
108
|
mannKendall: () => mannKendall,
|
|
107
109
|
mannKendallVerdict: () => mannKendallVerdict,
|
|
108
110
|
median: () => median,
|
|
@@ -155,8 +157,8 @@ function createLogger(moduleName) {
|
|
|
155
157
|
}
|
|
156
158
|
}();
|
|
157
159
|
}
|
|
158
|
-
function setObserverLogger(
|
|
159
|
-
mainLogger =
|
|
160
|
+
function setObserverLogger(logger10) {
|
|
161
|
+
mainLogger = logger10;
|
|
160
162
|
}
|
|
161
163
|
|
|
162
164
|
// src/ObservedCall.ts
|
|
@@ -3574,7 +3576,6 @@ var UnconsumedTrackDetector = class _UnconsumedTrackDetector {
|
|
|
3574
3576
|
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
3575
3577
|
timestamp: now,
|
|
3576
3578
|
payload: {
|
|
3577
|
-
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
3578
3579
|
trackId: outboundTrack.id,
|
|
3579
3580
|
kind: outboundTrack.kind,
|
|
3580
3581
|
publisherClientId: peerConnection?.client.clientId,
|
|
@@ -3720,7 +3721,6 @@ var TrackDeliveryMismatchDetector = class _TrackDeliveryMismatchDetector {
|
|
|
3720
3721
|
type,
|
|
3721
3722
|
timestamp: now,
|
|
3722
3723
|
payload: {
|
|
3723
|
-
type,
|
|
3724
3724
|
trackId: outboundTrackId,
|
|
3725
3725
|
publisherClientId: delivery.publisherClientId,
|
|
3726
3726
|
publisherSending: delivery.publisherSending,
|
|
@@ -3973,12 +3973,9 @@ var CallConcurrentIssueDetector = class _CallConcurrentIssueDetector {
|
|
|
3973
3973
|
this._call.addIssue({
|
|
3974
3974
|
type: issueType,
|
|
3975
3975
|
timestamp: now,
|
|
3976
|
+
conclusion,
|
|
3976
3977
|
payload: {
|
|
3977
|
-
type: issueType,
|
|
3978
3978
|
issueType: type,
|
|
3979
|
-
scope: "call",
|
|
3980
|
-
conclusion,
|
|
3981
|
-
callId: this._call.callId,
|
|
3982
3979
|
clients: group.totalClients,
|
|
3983
3980
|
affectedClients: group.clientIds.length,
|
|
3984
3981
|
affectedRatio: group.affectedRatio,
|
|
@@ -4102,10 +4099,9 @@ var IssueFanOutDetector = class _IssueFanOutDetector {
|
|
|
4102
4099
|
this._call.addIssue({
|
|
4103
4100
|
type,
|
|
4104
4101
|
timestamp: now,
|
|
4102
|
+
conclusion,
|
|
4105
4103
|
payload: {
|
|
4106
|
-
type,
|
|
4107
4104
|
issueType,
|
|
4108
|
-
conclusion,
|
|
4109
4105
|
trackId: publisher.id,
|
|
4110
4106
|
kind: publisher.kind,
|
|
4111
4107
|
publisherClientId: publisher.getPeerConnection().client.clientId,
|
|
@@ -4252,18 +4248,14 @@ var PublisherFaultCorroborationDetector = class _PublisherFaultCorroborationDete
|
|
|
4252
4248
|
this._call.addIssue({
|
|
4253
4249
|
type: PublisherFaultTypes.corroboratedPublisherFault,
|
|
4254
4250
|
timestamp: now,
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
|
|
4263
|
-
// Higher than any single-ended finding: two independent parties, one conclusion.
|
|
4264
|
-
confidence: 0.9
|
|
4265
|
-
}
|
|
4266
|
-
}
|
|
4251
|
+
conclusion: {
|
|
4252
|
+
faultDomain: "published-track",
|
|
4253
|
+
summary: `${fault.publisherClientId} reports ${fault.publisherIssueTypes.join(", ")} on track ${fault.trackId} while ${fault.affectedReceivers} of ${fault.receivers} subscribers report ${fault.receiverIssueTypes.join(", ")} \u2014 both ends agree`,
|
|
4254
|
+
recommendation: "the source is implicated, not inferred: check that publisher's capture, encoder and uplink before looking at the SFU or the receivers",
|
|
4255
|
+
// Higher than any single-ended finding: two independent parties, one conclusion.
|
|
4256
|
+
confidence: 0.9
|
|
4257
|
+
},
|
|
4258
|
+
payload: { ...fault }
|
|
4267
4259
|
});
|
|
4268
4260
|
}
|
|
4269
4261
|
}
|
|
@@ -4319,6 +4311,14 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
4319
4311
|
value: void 0
|
|
4320
4312
|
};
|
|
4321
4313
|
remoteTrackResolver;
|
|
4314
|
+
/**
|
|
4315
|
+
* The accumulating record of this call's life, or `undefined` when no summary was configured.
|
|
4316
|
+
*
|
|
4317
|
+
* Live — read it at any point during the call. It is also delivered once on `call-summary` when
|
|
4318
|
+
* the call closes. See `CallSummary`: an absent section means "not collected", never "nothing
|
|
4319
|
+
* happened".
|
|
4320
|
+
*/
|
|
4321
|
+
summary;
|
|
4322
4322
|
/**
|
|
4323
4323
|
* Published tracks that currently have **no** subscriber linked to them.
|
|
4324
4324
|
*
|
|
@@ -4396,6 +4396,21 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
4396
4396
|
this.detectors.add(detector);
|
|
4397
4397
|
return this;
|
|
4398
4398
|
}
|
|
4399
|
+
/**
|
|
4400
|
+
* Start accumulating this call's summary, if the observer was configured for summaries.
|
|
4401
|
+
*
|
|
4402
|
+
* Called by `createObservedCall`; you should not need it. It takes no configuration of its own on
|
|
4403
|
+
* purpose: the collector subscribes to exactly the events the observer's `include` requires, so a
|
|
4404
|
+
* per-call section outside that set would be created and then never written to — an empty section
|
|
4405
|
+
* that reads as "nothing happened". One shape per observer is the only shape that can be filled.
|
|
4406
|
+
*
|
|
4407
|
+
* The collector builds it rather than this method, so the resolved configuration never has to
|
|
4408
|
+
* leave the one object that owns it. Returns `undefined` when summaries are off, and is
|
|
4409
|
+
* idempotent: an existing summary is kept, not restarted.
|
|
4410
|
+
*/
|
|
4411
|
+
enableSummary() {
|
|
4412
|
+
return this.summary ??= this.observer.callSummaryCollector?.createSummary(this.callId);
|
|
4413
|
+
}
|
|
4399
4414
|
/**
|
|
4400
4415
|
* Remove a detector from **this call** by name, returning how many were removed.
|
|
4401
4416
|
*
|
|
@@ -4420,12 +4435,13 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
4420
4435
|
/**
|
|
4421
4436
|
* Raise a call-level (server-side) finding; surfaced on the Observer bus as `call-issue`.
|
|
4422
4437
|
*
|
|
4423
|
-
* `payload`
|
|
4424
|
-
* serialise for.
|
|
4438
|
+
* `payload` is an **object** and holds evidence only — it is delivered to an in-process handler,
|
|
4439
|
+
* so there is nothing to serialise for. `scope` is stamped here, and the `callId` is already on
|
|
4440
|
+
* the event, so neither belongs in the payload. Put the interpretation in `conclusion`.
|
|
4425
4441
|
*/
|
|
4426
4442
|
addIssue(issue) {
|
|
4427
4443
|
if (this.closed) return;
|
|
4428
|
-
this._notify("call-issue", { ...this.eventScope, issue });
|
|
4444
|
+
this._notify("call-issue", { ...this.eventScope, issue: { ...issue, scope: "call" } });
|
|
4429
4445
|
}
|
|
4430
4446
|
close() {
|
|
4431
4447
|
if (this.closed) return;
|
|
@@ -4446,6 +4462,10 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
4446
4462
|
if (this.startedAt === void 0) this.startedAt = minSampleTimestamps;
|
|
4447
4463
|
if (this.endedAt === void 0) this.endedAt = maxSampleTimestamps;
|
|
4448
4464
|
this.closedAt = Date.now();
|
|
4465
|
+
if (this.summary) {
|
|
4466
|
+
this.observer.callSummaryCollector?.finalise(this);
|
|
4467
|
+
this._notify("call-summary", { ...this.eventScope, summary: this.summary });
|
|
4468
|
+
}
|
|
4449
4469
|
this.detectors.clear();
|
|
4450
4470
|
this.activeIssuesRegistry.clear();
|
|
4451
4471
|
this.emit("close");
|
|
@@ -5206,6 +5226,166 @@ var ActiveIssuesRegistry = class {
|
|
|
5206
5226
|
}
|
|
5207
5227
|
};
|
|
5208
5228
|
|
|
5229
|
+
// src/summaries/CallSummary.ts
|
|
5230
|
+
var defaultCallSummaryConfig = {
|
|
5231
|
+
include: [],
|
|
5232
|
+
maxIssues: 500,
|
|
5233
|
+
maxClientIds: 1e4
|
|
5234
|
+
};
|
|
5235
|
+
function createCallSummary(callId, config) {
|
|
5236
|
+
const summary = { callId, attachments: {} };
|
|
5237
|
+
if (config.include.includes("clients")) {
|
|
5238
|
+
summary.clients = { clientIds: [], peak: 0, joined: 0, left: 0 };
|
|
5239
|
+
}
|
|
5240
|
+
if (config.include.includes("issues")) {
|
|
5241
|
+
summary.issues = [];
|
|
5242
|
+
}
|
|
5243
|
+
if (config.include.includes("turnServers")) {
|
|
5244
|
+
summary.turnServers = { serverUrls: [], clientsRelayed: 0 };
|
|
5245
|
+
}
|
|
5246
|
+
if (config.include.includes("scores")) {
|
|
5247
|
+
summary.scores = { samples: 0 };
|
|
5248
|
+
}
|
|
5249
|
+
return summary;
|
|
5250
|
+
}
|
|
5251
|
+
|
|
5252
|
+
// src/summaries/CallSummaryCollector.ts
|
|
5253
|
+
var logger8 = createLogger("CallSummaryCollector");
|
|
5254
|
+
var CallSummaryCollector = class {
|
|
5255
|
+
constructor(_observer, _config) {
|
|
5256
|
+
this._observer = _observer;
|
|
5257
|
+
this._config = _config;
|
|
5258
|
+
this._subscribeBuiltIns();
|
|
5259
|
+
this._subscribeEnrichers();
|
|
5260
|
+
}
|
|
5261
|
+
_observer;
|
|
5262
|
+
_config;
|
|
5263
|
+
_scratch = /* @__PURE__ */ new WeakMap();
|
|
5264
|
+
_listeners = [];
|
|
5265
|
+
_closed = false;
|
|
5266
|
+
/**
|
|
5267
|
+
* Build a summary for `callId` and start tracking it.
|
|
5268
|
+
*
|
|
5269
|
+
* Creating it here, rather than letting the call create one and hand it over, keeps the resolved
|
|
5270
|
+
* configuration inside the single object that owns it — and makes it impossible to end up with a
|
|
5271
|
+
* summary whose sections nobody subscribed to fill.
|
|
5272
|
+
*/
|
|
5273
|
+
createSummary(callId) {
|
|
5274
|
+
const summary = createCallSummary(callId, this._config);
|
|
5275
|
+
this._scratch.set(summary, { scores: [], turnClientIds: /* @__PURE__ */ new Set() });
|
|
5276
|
+
return summary;
|
|
5277
|
+
}
|
|
5278
|
+
/**
|
|
5279
|
+
* Finalise `call`'s summary: fold in what only makes sense once, and stamp the closing times.
|
|
5280
|
+
*
|
|
5281
|
+
* Percentiles are computed here rather than on every update — a median recomputed per tick over a
|
|
5282
|
+
* growing array is quadratic work to produce a number nobody reads until the end.
|
|
5283
|
+
*/
|
|
5284
|
+
finalise(call) {
|
|
5285
|
+
const summary = call.summary;
|
|
5286
|
+
if (!summary) return;
|
|
5287
|
+
const scratch = this._scratch.get(summary);
|
|
5288
|
+
summary.startedAt = call.startedAt;
|
|
5289
|
+
summary.endedAt = call.endedAt;
|
|
5290
|
+
summary.durationInMs = summary.startedAt !== void 0 && summary.endedAt !== void 0 ? Math.max(0, summary.endedAt - summary.startedAt) : void 0;
|
|
5291
|
+
summary.closedAt = Date.now();
|
|
5292
|
+
if (summary.scores && scratch) {
|
|
5293
|
+
summary.scores.samples = scratch.scores.length;
|
|
5294
|
+
if (0 < scratch.scores.length) {
|
|
5295
|
+
summary.scores.min = Math.min(...scratch.scores);
|
|
5296
|
+
summary.scores.max = Math.max(...scratch.scores);
|
|
5297
|
+
summary.scores.median = percentile(scratch.scores, 0.5);
|
|
5298
|
+
}
|
|
5299
|
+
}
|
|
5300
|
+
if (summary.turnServers && scratch) {
|
|
5301
|
+
summary.turnServers.clientsRelayed = scratch.turnClientIds.size;
|
|
5302
|
+
}
|
|
5303
|
+
}
|
|
5304
|
+
/** Drop every bus subscription. Called when the observer closes. */
|
|
5305
|
+
close() {
|
|
5306
|
+
if (this._closed) return;
|
|
5307
|
+
this._closed = true;
|
|
5308
|
+
for (const { event, listener } of this._listeners) {
|
|
5309
|
+
this._observer.off(event, listener);
|
|
5310
|
+
}
|
|
5311
|
+
this._listeners.length = 0;
|
|
5312
|
+
}
|
|
5313
|
+
/**
|
|
5314
|
+
* Subscribe `listener` to `event`, routed to the summary of the call the event names.
|
|
5315
|
+
*
|
|
5316
|
+
* The `observedCall` is read off the payload rather than closed over, which is what lets one
|
|
5317
|
+
* subscription serve every call.
|
|
5318
|
+
*/
|
|
5319
|
+
_on(event, handler) {
|
|
5320
|
+
const listener = (...args) => {
|
|
5321
|
+
const summary = args[0].observedCall.summary;
|
|
5322
|
+
if (!summary) return;
|
|
5323
|
+
const scratch = this._scratch.get(summary);
|
|
5324
|
+
if (!scratch) return;
|
|
5325
|
+
try {
|
|
5326
|
+
handler(summary, scratch, ...args);
|
|
5327
|
+
} catch (err) {
|
|
5328
|
+
logger8.warn("A call-summary handler for %s threw; continuing. %o", event, err);
|
|
5329
|
+
}
|
|
5330
|
+
};
|
|
5331
|
+
this._observer.on(event, listener);
|
|
5332
|
+
this._listeners.push({ event, listener });
|
|
5333
|
+
}
|
|
5334
|
+
_subscribeBuiltIns() {
|
|
5335
|
+
const include = this._config.include;
|
|
5336
|
+
if (include.includes("clients")) {
|
|
5337
|
+
this._on("client-added", (summary, _scratch, { observedCall, observedClient }) => {
|
|
5338
|
+
const clients = summary.clients;
|
|
5339
|
+
if (!clients) return;
|
|
5340
|
+
clients.joined += 1;
|
|
5341
|
+
clients.peak = Math.max(clients.peak, observedCall.observedClients.size);
|
|
5342
|
+
if (clients.clientIds.length < this._config.maxClientIds) {
|
|
5343
|
+
clients.clientIds.push(observedClient.clientId);
|
|
5344
|
+
} else {
|
|
5345
|
+
summary.truncated = { ...summary.truncated, clientIds: (summary.truncated?.clientIds ?? 0) + 1 };
|
|
5346
|
+
}
|
|
5347
|
+
});
|
|
5348
|
+
this._on("client-closed", (summary) => {
|
|
5349
|
+
if (summary.clients) summary.clients.left += 1;
|
|
5350
|
+
});
|
|
5351
|
+
}
|
|
5352
|
+
if (include.includes("issues")) {
|
|
5353
|
+
this._on("call-issue", (summary, _scratch, { issue }) => {
|
|
5354
|
+
const issues = summary.issues;
|
|
5355
|
+
if (!issues) return;
|
|
5356
|
+
if (issues.length < this._config.maxIssues) issues.push(issue);
|
|
5357
|
+
else summary.truncated = { ...summary.truncated, issues: (summary.truncated?.issues ?? 0) + 1 };
|
|
5358
|
+
});
|
|
5359
|
+
}
|
|
5360
|
+
if (include.includes("scores") || include.includes("turnServers")) {
|
|
5361
|
+
this._on("call-updated", (summary, scratch, { observedCall }) => {
|
|
5362
|
+
if (summary.scores && observedCall.score !== void 0) scratch.scores.push(observedCall.score);
|
|
5363
|
+
if (!summary.turnServers) return;
|
|
5364
|
+
for (const clientId of observedCall.clientsUsedTurn) scratch.turnClientIds.add(clientId);
|
|
5365
|
+
for (const client of observedCall.observedClients.values()) {
|
|
5366
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
5367
|
+
for (const pair of peerConnection.selectedIceCandiadtePairForTurn) {
|
|
5368
|
+
const url = pair.getLocalCandidate()?.url;
|
|
5369
|
+
if (url && !summary.turnServers.serverUrls.includes(url)) {
|
|
5370
|
+
summary.turnServers.serverUrls.push(url);
|
|
5371
|
+
}
|
|
5372
|
+
}
|
|
5373
|
+
}
|
|
5374
|
+
}
|
|
5375
|
+
});
|
|
5376
|
+
}
|
|
5377
|
+
}
|
|
5378
|
+
_subscribeEnrichers() {
|
|
5379
|
+
const enrich = this._config.enrich;
|
|
5380
|
+
if (!enrich) return;
|
|
5381
|
+
for (const name of Object.keys(enrich)) {
|
|
5382
|
+
const enricher = enrich[name];
|
|
5383
|
+
if (!enricher) continue;
|
|
5384
|
+
this._on(name, (summary, _scratch, ...args) => enricher(summary, ...args));
|
|
5385
|
+
}
|
|
5386
|
+
}
|
|
5387
|
+
};
|
|
5388
|
+
|
|
5209
5389
|
// src/detectors/SfuCongestionDetector.ts
|
|
5210
5390
|
var SfuCongestionDetector = class _SfuCongestionDetector {
|
|
5211
5391
|
constructor(_observer, config = {}) {
|
|
@@ -5334,13 +5514,10 @@ var SfuCongestionDetector = class _SfuCongestionDetector {
|
|
|
5334
5514
|
absoluteIncrease: evaluation.absoluteIncrease,
|
|
5335
5515
|
relativeIncrease: evaluation.relativeIncrease
|
|
5336
5516
|
};
|
|
5337
|
-
this._observer.
|
|
5338
|
-
|
|
5339
|
-
|
|
5340
|
-
|
|
5341
|
-
payload
|
|
5342
|
-
},
|
|
5343
|
-
observer: this._observer
|
|
5517
|
+
this._observer.addIssue({
|
|
5518
|
+
type: this._config.emittedObserverIssueType,
|
|
5519
|
+
timestamp: Date.now(),
|
|
5520
|
+
payload: { ...payload }
|
|
5344
5521
|
});
|
|
5345
5522
|
}
|
|
5346
5523
|
/**
|
|
@@ -5454,11 +5631,9 @@ var ObserverConcurrentIssueDetector = class _ObserverConcurrentIssueDetector {
|
|
|
5454
5631
|
this._observer.addIssue({
|
|
5455
5632
|
type: issueType,
|
|
5456
5633
|
timestamp: now,
|
|
5634
|
+
conclusion,
|
|
5457
5635
|
payload: {
|
|
5458
|
-
type: issueType,
|
|
5459
5636
|
issueType: type,
|
|
5460
|
-
scope: "observer",
|
|
5461
|
-
conclusion,
|
|
5462
5637
|
clients: group.totalClients,
|
|
5463
5638
|
affectedClients: group.clientIds.length,
|
|
5464
5639
|
affectedRatio: group.affectedRatio,
|
|
@@ -5631,16 +5806,13 @@ var ClientPopulationIssueDetector = class _ClientPopulationIssueDetector {
|
|
|
5631
5806
|
this._observer.addIssue({
|
|
5632
5807
|
type: ClientPopulationIssueTypes.clientPopulationIssue,
|
|
5633
5808
|
timestamp: now,
|
|
5634
|
-
|
|
5635
|
-
|
|
5636
|
-
|
|
5637
|
-
|
|
5638
|
-
|
|
5639
|
-
|
|
5640
|
-
|
|
5641
|
-
confidence: this._confidenceOf(rollup)
|
|
5642
|
-
}
|
|
5643
|
-
}
|
|
5809
|
+
conclusion: {
|
|
5810
|
+
faultDomain: "client-population",
|
|
5811
|
+
summary: `'${issueType}' is ${this._riskText(rollup.relativeRisk)} more likely on ${population} than on the rest of the fleet (${rollup.affectedClients}/${rollup.clients} vs ${rollup.controlAffectedClients}/${rollup.controlClients})`,
|
|
5812
|
+
recommendation: "this is not an SFU symptom \u2014 look at what those clients share: a recent release, a browser version, or shared/virtualised hardware",
|
|
5813
|
+
confidence: this._confidenceOf(rollup)
|
|
5814
|
+
},
|
|
5815
|
+
payload: { ...rollup }
|
|
5644
5816
|
});
|
|
5645
5817
|
}
|
|
5646
5818
|
}
|
|
@@ -5742,7 +5914,7 @@ var TurnServerHealthDetector = class _TurnServerHealthDetector {
|
|
|
5742
5914
|
this._observer.addIssue({
|
|
5743
5915
|
type: TurnServerHealthTypes.turnServerDegraded,
|
|
5744
5916
|
timestamp: now,
|
|
5745
|
-
payload: {
|
|
5917
|
+
payload: { ...health, otherServers }
|
|
5746
5918
|
});
|
|
5747
5919
|
}
|
|
5748
5920
|
for (const serverUrl of [...this._streaks.keys()]) {
|
|
@@ -5850,7 +6022,6 @@ var TurnServerOutageDetector = class _TurnServerOutageDetector {
|
|
|
5850
6022
|
type: TurnServerOutageTypes.turnServerOutage,
|
|
5851
6023
|
timestamp: now,
|
|
5852
6024
|
payload: {
|
|
5853
|
-
type: TurnServerOutageTypes.turnServerOutage,
|
|
5854
6025
|
serverUrl,
|
|
5855
6026
|
peakClients: peak,
|
|
5856
6027
|
currentClients: live,
|
|
@@ -6124,17 +6295,13 @@ var SimulcastReceiverValidator = class _SimulcastReceiverValidator {
|
|
|
6124
6295
|
this._observer.addIssue({
|
|
6125
6296
|
type: LOWEST_COMMON_DENOMINATOR_ISSUE,
|
|
6126
6297
|
timestamp: decidedAt,
|
|
6127
|
-
|
|
6128
|
-
|
|
6129
|
-
|
|
6130
|
-
|
|
6131
|
-
|
|
6132
|
-
|
|
6133
|
-
|
|
6134
|
-
recommendation: "check that simulcast/SVC is enabled and layers are chosen per consumer, and that the SFU terminates receiver reports instead of forwarding them; this is a build/config property, not a transient",
|
|
6135
|
-
confidence: 0.8
|
|
6136
|
-
}
|
|
6137
|
-
}
|
|
6298
|
+
conclusion: {
|
|
6299
|
+
faultDomain: "infrastructure",
|
|
6300
|
+
summary: "one bad receiver is dragging a publisher's bitrate down for everyone \u2014 the SFU is not selecting layers per consumer",
|
|
6301
|
+
recommendation: "check that simulcast/SVC is enabled and layers are chosen per consumer, and that the SFU terminates receiver reports instead of forwarding them; this is a build/config property, not a transient",
|
|
6302
|
+
confidence: 0.8
|
|
6303
|
+
},
|
|
6304
|
+
payload: { checks: this._checks, ...outcome.evidence }
|
|
6138
6305
|
});
|
|
6139
6306
|
}
|
|
6140
6307
|
this.onDone(this.report);
|
|
@@ -6237,17 +6404,13 @@ var RemoteTrackResolverValidator = class _RemoteTrackResolverValidator {
|
|
|
6237
6404
|
this._observer.addIssue({
|
|
6238
6405
|
type: UNRESOLVED_TRACK_LINKS_ISSUE,
|
|
6239
6406
|
timestamp: decidedAt,
|
|
6240
|
-
|
|
6241
|
-
|
|
6242
|
-
|
|
6243
|
-
|
|
6244
|
-
|
|
6245
|
-
|
|
6246
|
-
|
|
6247
|
-
recommendation: "check the id the resolver joins on (mediasoup producerId, or your own convention) \u2014 until it links, IssueFanOutDetector, TrackDeliveryMismatchDetector, UnconsumedTrackDetector and SimulcastReceiverValidator all silently do nothing",
|
|
6248
|
-
confidence: 0.9
|
|
6249
|
-
}
|
|
6250
|
-
}
|
|
6407
|
+
conclusion: {
|
|
6408
|
+
faultDomain: "infrastructure",
|
|
6409
|
+
summary: "a RemoteTrackResolver is configured but has never linked a subscriber to a publisher",
|
|
6410
|
+
recommendation: "check the id the resolver joins on (mediasoup producerId, or your own convention) \u2014 until it links, IssueFanOutDetector, TrackDeliveryMismatchDetector, UnconsumedTrackDetector and SimulcastReceiverValidator all silently do nothing",
|
|
6411
|
+
confidence: 0.9
|
|
6412
|
+
},
|
|
6413
|
+
payload: { checks: this._checks, ...outcome.evidence }
|
|
6251
6414
|
});
|
|
6252
6415
|
}
|
|
6253
6416
|
this.onDone(this.report);
|
|
@@ -6358,17 +6521,16 @@ var CodecConsistencyValidator = class _CodecConsistencyValidator {
|
|
|
6358
6521
|
this._observer.addIssue({
|
|
6359
6522
|
type: CODEC_MISMATCH_ISSUE,
|
|
6360
6523
|
timestamp: decidedAt,
|
|
6524
|
+
conclusion: {
|
|
6525
|
+
faultDomain: "infrastructure",
|
|
6526
|
+
summary: outcome.verdict === "codec-split" ? "participants of one call are using different codecs \u2014 an SFU that forwards without transcoding cannot serve all of them" : "the deployment is consistently negotiating a codec other than the expected one",
|
|
6527
|
+
recommendation: outcome.verdict === "codec-split" ? "check codec preferences and any SDP munging; a split usually means one endpoint could not negotiate the preferred codec and the others were not renegotiated with it" : "check codec preferences and endpoint support \u2014 a silent fallback keeps working, at a higher bitrate than you budgeted for",
|
|
6528
|
+
confidence: 0.85
|
|
6529
|
+
},
|
|
6361
6530
|
payload: {
|
|
6362
|
-
type: CODEC_MISMATCH_ISSUE,
|
|
6363
6531
|
verdict: outcome.verdict,
|
|
6364
6532
|
checks: this._checks,
|
|
6365
|
-
evidence: outcome.evidence
|
|
6366
|
-
conclusion: {
|
|
6367
|
-
faultDomain: "infrastructure",
|
|
6368
|
-
summary: outcome.verdict === "codec-split" ? "participants of one call are using different codecs \u2014 an SFU that forwards without transcoding cannot serve all of them" : "the deployment is consistently negotiating a codec other than the expected one",
|
|
6369
|
-
recommendation: outcome.verdict === "codec-split" ? "check codec preferences and any SDP munging; a split usually means one endpoint could not negotiate the preferred codec and the others were not renegotiated with it" : "check codec preferences and endpoint support \u2014 a silent fallback keeps working, at a higher bitrate than you budgeted for",
|
|
6370
|
-
confidence: 0.85
|
|
6371
|
-
}
|
|
6533
|
+
evidence: outcome.evidence
|
|
6372
6534
|
}
|
|
6373
6535
|
});
|
|
6374
6536
|
}
|
|
@@ -6382,7 +6544,7 @@ function mediaKindOf(mimeType) {
|
|
|
6382
6544
|
}
|
|
6383
6545
|
|
|
6384
6546
|
// src/Observer.ts
|
|
6385
|
-
var
|
|
6547
|
+
var logger9 = createLogger("Observer");
|
|
6386
6548
|
var Observer = class extends import_events6.EventEmitter {
|
|
6387
6549
|
observedTURN = new ObservedTURN();
|
|
6388
6550
|
observedCalls = /* @__PURE__ */ new Map();
|
|
@@ -6438,6 +6600,14 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6438
6600
|
* ```
|
|
6439
6601
|
*/
|
|
6440
6602
|
callDetectorConfigs = /* @__PURE__ */ new Map();
|
|
6603
|
+
/**
|
|
6604
|
+
* Owns every call's summary: the resolved `config.callSummary`, the bus subscriptions that keep
|
|
6605
|
+
* the summaries current (one per event type, not one per call), and the summaries themselves.
|
|
6606
|
+
*
|
|
6607
|
+
* `undefined` when `config.callSummary` was absent or `null` — so its presence *is* the answer to
|
|
6608
|
+
* "are summaries on", and nothing is subscribed to anything.
|
|
6609
|
+
*/
|
|
6610
|
+
callSummaryCollector;
|
|
6441
6611
|
constructor(config = {}) {
|
|
6442
6612
|
super();
|
|
6443
6613
|
this.setMaxListeners(Infinity);
|
|
@@ -6449,6 +6619,12 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6449
6619
|
closeClientIfIdleForMs: 6e4,
|
|
6450
6620
|
...config
|
|
6451
6621
|
};
|
|
6622
|
+
if (this.config.callSummary) {
|
|
6623
|
+
this.callSummaryCollector = new CallSummaryCollector(this, {
|
|
6624
|
+
...defaultCallSummaryConfig,
|
|
6625
|
+
...this.config.callSummary
|
|
6626
|
+
});
|
|
6627
|
+
}
|
|
6452
6628
|
}
|
|
6453
6629
|
get numberOfCalls() {
|
|
6454
6630
|
return this.observedCalls.size;
|
|
@@ -6487,7 +6663,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6487
6663
|
break;
|
|
6488
6664
|
}
|
|
6489
6665
|
default: {
|
|
6490
|
-
|
|
6666
|
+
logger9.warn("Unknown detector name %s; skipping", name);
|
|
6491
6667
|
return this;
|
|
6492
6668
|
}
|
|
6493
6669
|
}
|
|
@@ -6572,7 +6748,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6572
6748
|
break;
|
|
6573
6749
|
}
|
|
6574
6750
|
if (!validator) {
|
|
6575
|
-
|
|
6751
|
+
logger9.warn("Unknown validator name %s; skipping", name);
|
|
6576
6752
|
return this;
|
|
6577
6753
|
}
|
|
6578
6754
|
this.validators.add(validator);
|
|
@@ -6603,7 +6779,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6603
6779
|
try {
|
|
6604
6780
|
validator.cancel(reason);
|
|
6605
6781
|
} catch (err) {
|
|
6606
|
-
|
|
6782
|
+
logger9.warn("Error cancelling validator %s: %o", validator.name, err);
|
|
6607
6783
|
}
|
|
6608
6784
|
}
|
|
6609
6785
|
return matching.length;
|
|
@@ -6614,11 +6790,11 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6614
6790
|
}
|
|
6615
6791
|
createObservedCall(settings) {
|
|
6616
6792
|
if (this.closed) {
|
|
6617
|
-
|
|
6793
|
+
logger9.warn("Attempted to create a call (callId: %s) on a closed observer", settings.callId);
|
|
6618
6794
|
return void 0;
|
|
6619
6795
|
}
|
|
6620
6796
|
if (this.observedCalls.has(settings.callId)) {
|
|
6621
|
-
|
|
6797
|
+
logger9.warn("Observed Call with id %s already exists; returning the existing instance", settings.callId);
|
|
6622
6798
|
return this.observedCalls.get(settings.callId);
|
|
6623
6799
|
}
|
|
6624
6800
|
const callSettings = {
|
|
@@ -6633,6 +6809,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6633
6809
|
callActiveIssuesRegistry
|
|
6634
6810
|
);
|
|
6635
6811
|
observedCall.remoteTrackResolver = this.config.createRemoteTrackResolver?.(observedCall);
|
|
6812
|
+
observedCall.enableSummary();
|
|
6636
6813
|
for (const [name, detectorConfig] of this.callDetectorConfigs) {
|
|
6637
6814
|
observedCall.addDetector(name, detectorConfig);
|
|
6638
6815
|
}
|
|
@@ -6655,11 +6832,11 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6655
6832
|
}
|
|
6656
6833
|
createObservedMediasoupRouter(settings) {
|
|
6657
6834
|
if (this.closed) {
|
|
6658
|
-
|
|
6835
|
+
logger9.warn("Attempted to create mediasoup router (id: %d) on a closed observer", settings.router.id);
|
|
6659
6836
|
return void 0;
|
|
6660
6837
|
}
|
|
6661
6838
|
if (this.observedMediasoupRouters.has(settings.router.id)) {
|
|
6662
|
-
|
|
6839
|
+
logger9.warn("Observed Mediasoup Router (id %s) already exists; returning the existing instance", settings.router.id);
|
|
6663
6840
|
return this.observedMediasoupRouters.get(settings.router.id);
|
|
6664
6841
|
}
|
|
6665
6842
|
const observedMediasoupRouter = new ObservedMediasoupRouter(settings);
|
|
@@ -6690,13 +6867,14 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6690
6867
|
}
|
|
6691
6868
|
close() {
|
|
6692
6869
|
if (this.closed) {
|
|
6693
|
-
return
|
|
6870
|
+
return logger9.debug("Attempted to close twice");
|
|
6694
6871
|
}
|
|
6695
6872
|
this.closed = true;
|
|
6696
6873
|
for (const call of [...this.observedCalls.values()]) call.close();
|
|
6697
6874
|
this.detectors.clear();
|
|
6698
6875
|
for (const validator of [...this.validators]) validator.cancel("observer closed");
|
|
6699
6876
|
this.validators.clear();
|
|
6877
|
+
this.callSummaryCollector?.close();
|
|
6700
6878
|
this.activeIssuesRegistry.clear();
|
|
6701
6879
|
this._notify("observer-closed", { ...this.eventScope });
|
|
6702
6880
|
}
|
|
@@ -6707,7 +6885,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6707
6885
|
try {
|
|
6708
6886
|
this.acceptMiddlewares.process({ sample, context });
|
|
6709
6887
|
} catch (err) {
|
|
6710
|
-
|
|
6888
|
+
logger9.warn("An accept middleware threw; dropping the sample. %o", err);
|
|
6711
6889
|
}
|
|
6712
6890
|
if (!sample.callId) {
|
|
6713
6891
|
this._notify("sample-rejected", { ...this.eventScope, reason: "missing-callId", sample });
|
|
@@ -6759,7 +6937,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6759
6937
|
try {
|
|
6760
6938
|
validator.update();
|
|
6761
6939
|
} catch (err) {
|
|
6762
|
-
|
|
6940
|
+
logger9.warn("Error running validator %s: %o", validator.name, err);
|
|
6763
6941
|
}
|
|
6764
6942
|
}
|
|
6765
6943
|
this._notify("observer-updated", { ...this.eventScope });
|
|
@@ -6772,7 +6950,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6772
6950
|
*/
|
|
6773
6951
|
addIssue(issue) {
|
|
6774
6952
|
if (this.closed) return;
|
|
6775
|
-
this._notify("observer-issue", { ...this.eventScope, issue });
|
|
6953
|
+
this._notify("observer-issue", { ...this.eventScope, issue: { ...issue, scope: "observer" } });
|
|
6776
6954
|
}
|
|
6777
6955
|
/** Emit an Observer-bus event. */
|
|
6778
6956
|
_notify(type, ...args) {
|
|
@@ -6890,18 +7068,11 @@ var CallHealthAggregator = class {
|
|
|
6890
7068
|
}
|
|
6891
7069
|
};
|
|
6892
7070
|
|
|
6893
|
-
// src/common/
|
|
6894
|
-
function issuePayloadOf(issue) {
|
|
6895
|
-
const payload = issue.payload;
|
|
6896
|
-
if (payload === void 0) return void 0;
|
|
6897
|
-
return typeof payload === "string" ? parseJsonObject(payload) : payload;
|
|
6898
|
-
}
|
|
7071
|
+
// src/common/Issue.ts
|
|
6899
7072
|
function issuePayloadAsString(issue) {
|
|
6900
|
-
|
|
6901
|
-
if (payload === void 0) return void 0;
|
|
6902
|
-
if (typeof payload === "string") return payload;
|
|
7073
|
+
if (issue.payload === void 0) return void 0;
|
|
6903
7074
|
try {
|
|
6904
|
-
return JSON.stringify(payload);
|
|
7075
|
+
return JSON.stringify(issue.payload);
|
|
6905
7076
|
} catch {
|
|
6906
7077
|
return void 0;
|
|
6907
7078
|
}
|
|
@@ -7196,6 +7367,7 @@ function createP2pRemoteTrackResolverFactory() {
|
|
|
7196
7367
|
CallConcurrentIssueDetector,
|
|
7197
7368
|
CallConcurrentIssueTypes,
|
|
7198
7369
|
CallHealthAggregator,
|
|
7370
|
+
CallSummaryCollector,
|
|
7199
7371
|
ClientEventTypes,
|
|
7200
7372
|
ClientMetaTypes,
|
|
7201
7373
|
ClientPopulationIssueDetector,
|
|
@@ -7254,16 +7426,17 @@ function createP2pRemoteTrackResolverFactory() {
|
|
|
7254
7426
|
concludeObserverIssue,
|
|
7255
7427
|
correlation,
|
|
7256
7428
|
counterDelta,
|
|
7429
|
+
createCallSummary,
|
|
7257
7430
|
createDefaultMediasoupRemoteTrackResolverFactory,
|
|
7258
7431
|
createInMemorySink,
|
|
7259
7432
|
createJsonlFileSink,
|
|
7260
7433
|
createJsonlFileSinkFactory,
|
|
7261
7434
|
createLogger,
|
|
7262
7435
|
createP2pRemoteTrackResolverFactory,
|
|
7436
|
+
defaultCallSummaryConfig,
|
|
7263
7437
|
defaultClientHealthThresholds,
|
|
7264
7438
|
isClientIssueResolutionEntry,
|
|
7265
7439
|
issuePayloadAsString,
|
|
7266
|
-
issuePayloadOf,
|
|
7267
7440
|
mannKendall,
|
|
7268
7441
|
mannKendallVerdict,
|
|
7269
7442
|
median,
|