@observertc/observer-js 1.0.0-beta.16 → 1.0.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -34
- package/dist/index.d.mts +440 -139
- package/dist/index.d.ts +440 -139
- package/dist/index.js +298 -129
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +295 -128
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -37,8 +37,8 @@ function createLogger(moduleName) {
|
|
|
37
37
|
}
|
|
38
38
|
}();
|
|
39
39
|
}
|
|
40
|
-
function setObserverLogger(
|
|
41
|
-
mainLogger =
|
|
40
|
+
function setObserverLogger(logger10) {
|
|
41
|
+
mainLogger = logger10;
|
|
42
42
|
}
|
|
43
43
|
|
|
44
44
|
// src/ObservedCall.ts
|
|
@@ -3456,7 +3456,6 @@ var UnconsumedTrackDetector = class _UnconsumedTrackDetector {
|
|
|
3456
3456
|
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
3457
3457
|
timestamp: now,
|
|
3458
3458
|
payload: {
|
|
3459
|
-
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
3460
3459
|
trackId: outboundTrack.id,
|
|
3461
3460
|
kind: outboundTrack.kind,
|
|
3462
3461
|
publisherClientId: peerConnection?.client.clientId,
|
|
@@ -3602,7 +3601,6 @@ var TrackDeliveryMismatchDetector = class _TrackDeliveryMismatchDetector {
|
|
|
3602
3601
|
type,
|
|
3603
3602
|
timestamp: now,
|
|
3604
3603
|
payload: {
|
|
3605
|
-
type,
|
|
3606
3604
|
trackId: outboundTrackId,
|
|
3607
3605
|
publisherClientId: delivery.publisherClientId,
|
|
3608
3606
|
publisherSending: delivery.publisherSending,
|
|
@@ -3855,12 +3853,9 @@ var CallConcurrentIssueDetector = class _CallConcurrentIssueDetector {
|
|
|
3855
3853
|
this._call.addIssue({
|
|
3856
3854
|
type: issueType,
|
|
3857
3855
|
timestamp: now,
|
|
3856
|
+
conclusion,
|
|
3858
3857
|
payload: {
|
|
3859
|
-
type: issueType,
|
|
3860
3858
|
issueType: type,
|
|
3861
|
-
scope: "call",
|
|
3862
|
-
conclusion,
|
|
3863
|
-
callId: this._call.callId,
|
|
3864
3859
|
clients: group.totalClients,
|
|
3865
3860
|
affectedClients: group.clientIds.length,
|
|
3866
3861
|
affectedRatio: group.affectedRatio,
|
|
@@ -3984,10 +3979,9 @@ var IssueFanOutDetector = class _IssueFanOutDetector {
|
|
|
3984
3979
|
this._call.addIssue({
|
|
3985
3980
|
type,
|
|
3986
3981
|
timestamp: now,
|
|
3982
|
+
conclusion,
|
|
3987
3983
|
payload: {
|
|
3988
|
-
type,
|
|
3989
3984
|
issueType,
|
|
3990
|
-
conclusion,
|
|
3991
3985
|
trackId: publisher.id,
|
|
3992
3986
|
kind: publisher.kind,
|
|
3993
3987
|
publisherClientId: publisher.getPeerConnection().client.clientId,
|
|
@@ -4134,18 +4128,14 @@ var PublisherFaultCorroborationDetector = class _PublisherFaultCorroborationDete
|
|
|
4134
4128
|
this._call.addIssue({
|
|
4135
4129
|
type: PublisherFaultTypes.corroboratedPublisherFault,
|
|
4136
4130
|
timestamp: now,
|
|
4137
|
-
|
|
4138
|
-
|
|
4139
|
-
|
|
4140
|
-
|
|
4141
|
-
|
|
4142
|
-
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
// Higher than any single-ended finding: two independent parties, one conclusion.
|
|
4146
|
-
confidence: 0.9
|
|
4147
|
-
}
|
|
4148
|
-
}
|
|
4131
|
+
conclusion: {
|
|
4132
|
+
faultDomain: "published-track",
|
|
4133
|
+
summary: `${fault.publisherClientId} reports ${fault.publisherIssueTypes.join(", ")} on track ${fault.trackId} while ${fault.affectedReceivers} of ${fault.receivers} subscribers report ${fault.receiverIssueTypes.join(", ")} \u2014 both ends agree`,
|
|
4134
|
+
recommendation: "the source is implicated, not inferred: check that publisher's capture, encoder and uplink before looking at the SFU or the receivers",
|
|
4135
|
+
// Higher than any single-ended finding: two independent parties, one conclusion.
|
|
4136
|
+
confidence: 0.9
|
|
4137
|
+
},
|
|
4138
|
+
payload: { ...fault }
|
|
4149
4139
|
});
|
|
4150
4140
|
}
|
|
4151
4141
|
}
|
|
@@ -4201,6 +4191,14 @@ var ObservedCall = class extends EventEmitter3 {
|
|
|
4201
4191
|
value: void 0
|
|
4202
4192
|
};
|
|
4203
4193
|
remoteTrackResolver;
|
|
4194
|
+
/**
|
|
4195
|
+
* The accumulating record of this call's life, or `undefined` when no summary was configured.
|
|
4196
|
+
*
|
|
4197
|
+
* Live — read it at any point during the call. It is also delivered once on `call-summary` when
|
|
4198
|
+
* the call closes. See `CallSummary`: an absent section means "not collected", never "nothing
|
|
4199
|
+
* happened".
|
|
4200
|
+
*/
|
|
4201
|
+
summary;
|
|
4204
4202
|
/**
|
|
4205
4203
|
* Published tracks that currently have **no** subscriber linked to them.
|
|
4206
4204
|
*
|
|
@@ -4278,6 +4276,21 @@ var ObservedCall = class extends EventEmitter3 {
|
|
|
4278
4276
|
this.detectors.add(detector);
|
|
4279
4277
|
return this;
|
|
4280
4278
|
}
|
|
4279
|
+
/**
|
|
4280
|
+
* Start accumulating this call's summary, if the observer was configured for summaries.
|
|
4281
|
+
*
|
|
4282
|
+
* Called by `createObservedCall`; you should not need it. It takes no configuration of its own on
|
|
4283
|
+
* purpose: the collector subscribes to exactly the events the observer's `include` requires, so a
|
|
4284
|
+
* per-call section outside that set would be created and then never written to — an empty section
|
|
4285
|
+
* that reads as "nothing happened". One shape per observer is the only shape that can be filled.
|
|
4286
|
+
*
|
|
4287
|
+
* The collector builds it rather than this method, so the resolved configuration never has to
|
|
4288
|
+
* leave the one object that owns it. Returns `undefined` when summaries are off, and is
|
|
4289
|
+
* idempotent: an existing summary is kept, not restarted.
|
|
4290
|
+
*/
|
|
4291
|
+
enableSummary() {
|
|
4292
|
+
return this.summary ??= this.observer.callSummaryCollector?.createSummary(this.callId);
|
|
4293
|
+
}
|
|
4281
4294
|
/**
|
|
4282
4295
|
* Remove a detector from **this call** by name, returning how many were removed.
|
|
4283
4296
|
*
|
|
@@ -4302,12 +4315,13 @@ var ObservedCall = class extends EventEmitter3 {
|
|
|
4302
4315
|
/**
|
|
4303
4316
|
* Raise a call-level (server-side) finding; surfaced on the Observer bus as `call-issue`.
|
|
4304
4317
|
*
|
|
4305
|
-
* `payload`
|
|
4306
|
-
* serialise for.
|
|
4318
|
+
* `payload` is an **object** and holds evidence only — it is delivered to an in-process handler,
|
|
4319
|
+
* so there is nothing to serialise for. `scope` is stamped here, and the `callId` is already on
|
|
4320
|
+
* the event, so neither belongs in the payload. Put the interpretation in `conclusion`.
|
|
4307
4321
|
*/
|
|
4308
4322
|
addIssue(issue) {
|
|
4309
4323
|
if (this.closed) return;
|
|
4310
|
-
this._notify("call-issue", { ...this.eventScope, issue });
|
|
4324
|
+
this._notify("call-issue", { ...this.eventScope, issue: { ...issue, scope: "call" } });
|
|
4311
4325
|
}
|
|
4312
4326
|
close() {
|
|
4313
4327
|
if (this.closed) return;
|
|
@@ -4328,6 +4342,10 @@ var ObservedCall = class extends EventEmitter3 {
|
|
|
4328
4342
|
if (this.startedAt === void 0) this.startedAt = minSampleTimestamps;
|
|
4329
4343
|
if (this.endedAt === void 0) this.endedAt = maxSampleTimestamps;
|
|
4330
4344
|
this.closedAt = Date.now();
|
|
4345
|
+
if (this.summary) {
|
|
4346
|
+
this.observer.callSummaryCollector?.finalise(this);
|
|
4347
|
+
this._notify("call-summary", { ...this.eventScope, summary: this.summary });
|
|
4348
|
+
}
|
|
4331
4349
|
this.detectors.clear();
|
|
4332
4350
|
this.activeIssuesRegistry.clear();
|
|
4333
4351
|
this.emit("close");
|
|
@@ -4337,7 +4355,7 @@ var ObservedCall = class extends EventEmitter3 {
|
|
|
4337
4355
|
if (this.closed || !this.observedClients.has(clientId)) return;
|
|
4338
4356
|
return this.observedClients.get(clientId);
|
|
4339
4357
|
}
|
|
4340
|
-
createObservedClient(settings) {
|
|
4358
|
+
createObservedClient(settings, acceptCtx) {
|
|
4341
4359
|
if (this.closed) {
|
|
4342
4360
|
logger4.warn("Attempted to create a client (clientId: %s) on a closed call %s", settings.clientId, this.callId);
|
|
4343
4361
|
return void 0;
|
|
@@ -4346,15 +4364,18 @@ var ObservedCall = class extends EventEmitter3 {
|
|
|
4346
4364
|
logger4.warn("Client with id %s already exists in call %s; returning the existing instance", settings.clientId, this.callId);
|
|
4347
4365
|
return this.observedClients.get(settings.clientId);
|
|
4348
4366
|
}
|
|
4349
|
-
|
|
4350
|
-
settings
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4367
|
+
const clientSettings = {
|
|
4368
|
+
...settings,
|
|
4369
|
+
closeClientIfIdleForMs: settings.closeClientIfIdleForMs || this.observer.config.closeClientIfIdleForMs,
|
|
4370
|
+
appData: settings.appData ?? this.observer.config.createClientAppData?.({
|
|
4371
|
+
clientId: settings.clientId,
|
|
4372
|
+
observedCall: this,
|
|
4373
|
+
acceptCtx
|
|
4374
|
+
})
|
|
4375
|
+
};
|
|
4355
4376
|
const observedClientIssueRegistry = new ObservedClientIssueRegistry(this.activeIssuesRegistry);
|
|
4356
4377
|
const result = new ObservedClient(
|
|
4357
|
-
|
|
4378
|
+
clientSettings,
|
|
4358
4379
|
this,
|
|
4359
4380
|
observedClientIssueRegistry
|
|
4360
4381
|
);
|
|
@@ -4401,8 +4422,8 @@ var ObservedCall = class extends EventEmitter3 {
|
|
|
4401
4422
|
}
|
|
4402
4423
|
return result;
|
|
4403
4424
|
}
|
|
4404
|
-
getOrCreateObservedClient(settings) {
|
|
4405
|
-
return this.getObservedClient(settings.clientId) ?? this.createObservedClient(settings);
|
|
4425
|
+
getOrCreateObservedClient(settings, acceptCtx) {
|
|
4426
|
+
return this.getObservedClient(settings.clientId) ?? this.createObservedClient(settings, acceptCtx);
|
|
4406
4427
|
}
|
|
4407
4428
|
update(context) {
|
|
4408
4429
|
if (this.closed) return;
|
|
@@ -5088,6 +5109,166 @@ var ActiveIssuesRegistry = class {
|
|
|
5088
5109
|
}
|
|
5089
5110
|
};
|
|
5090
5111
|
|
|
5112
|
+
// src/summaries/CallSummary.ts
|
|
5113
|
+
var defaultCallSummaryConfig = {
|
|
5114
|
+
include: [],
|
|
5115
|
+
maxIssues: 500,
|
|
5116
|
+
maxClientIds: 1e4
|
|
5117
|
+
};
|
|
5118
|
+
function createCallSummary(callId, config) {
|
|
5119
|
+
const summary = { callId, attachments: {} };
|
|
5120
|
+
if (config.include.includes("clients")) {
|
|
5121
|
+
summary.clients = { clientIds: [], peak: 0, joined: 0, left: 0 };
|
|
5122
|
+
}
|
|
5123
|
+
if (config.include.includes("issues")) {
|
|
5124
|
+
summary.issues = [];
|
|
5125
|
+
}
|
|
5126
|
+
if (config.include.includes("turnServers")) {
|
|
5127
|
+
summary.turnServers = { serverUrls: [], clientsRelayed: 0 };
|
|
5128
|
+
}
|
|
5129
|
+
if (config.include.includes("scores")) {
|
|
5130
|
+
summary.scores = { samples: 0 };
|
|
5131
|
+
}
|
|
5132
|
+
return summary;
|
|
5133
|
+
}
|
|
5134
|
+
|
|
5135
|
+
// src/summaries/CallSummaryCollector.ts
|
|
5136
|
+
var logger8 = createLogger("CallSummaryCollector");
|
|
5137
|
+
var CallSummaryCollector = class {
|
|
5138
|
+
constructor(_observer, _config) {
|
|
5139
|
+
this._observer = _observer;
|
|
5140
|
+
this._config = _config;
|
|
5141
|
+
this._subscribeBuiltIns();
|
|
5142
|
+
this._subscribeEnrichers();
|
|
5143
|
+
}
|
|
5144
|
+
_observer;
|
|
5145
|
+
_config;
|
|
5146
|
+
_scratch = /* @__PURE__ */ new WeakMap();
|
|
5147
|
+
_listeners = [];
|
|
5148
|
+
_closed = false;
|
|
5149
|
+
/**
|
|
5150
|
+
* Build a summary for `callId` and start tracking it.
|
|
5151
|
+
*
|
|
5152
|
+
* Creating it here, rather than letting the call create one and hand it over, keeps the resolved
|
|
5153
|
+
* configuration inside the single object that owns it — and makes it impossible to end up with a
|
|
5154
|
+
* summary whose sections nobody subscribed to fill.
|
|
5155
|
+
*/
|
|
5156
|
+
createSummary(callId) {
|
|
5157
|
+
const summary = createCallSummary(callId, this._config);
|
|
5158
|
+
this._scratch.set(summary, { scores: [], turnClientIds: /* @__PURE__ */ new Set() });
|
|
5159
|
+
return summary;
|
|
5160
|
+
}
|
|
5161
|
+
/**
|
|
5162
|
+
* Finalise `call`'s summary: fold in what only makes sense once, and stamp the closing times.
|
|
5163
|
+
*
|
|
5164
|
+
* Percentiles are computed here rather than on every update — a median recomputed per tick over a
|
|
5165
|
+
* growing array is quadratic work to produce a number nobody reads until the end.
|
|
5166
|
+
*/
|
|
5167
|
+
finalise(call) {
|
|
5168
|
+
const summary = call.summary;
|
|
5169
|
+
if (!summary) return;
|
|
5170
|
+
const scratch = this._scratch.get(summary);
|
|
5171
|
+
summary.startedAt = call.startedAt;
|
|
5172
|
+
summary.endedAt = call.endedAt;
|
|
5173
|
+
summary.durationInMs = summary.startedAt !== void 0 && summary.endedAt !== void 0 ? Math.max(0, summary.endedAt - summary.startedAt) : void 0;
|
|
5174
|
+
summary.closedAt = Date.now();
|
|
5175
|
+
if (summary.scores && scratch) {
|
|
5176
|
+
summary.scores.samples = scratch.scores.length;
|
|
5177
|
+
if (0 < scratch.scores.length) {
|
|
5178
|
+
summary.scores.min = Math.min(...scratch.scores);
|
|
5179
|
+
summary.scores.max = Math.max(...scratch.scores);
|
|
5180
|
+
summary.scores.median = percentile(scratch.scores, 0.5);
|
|
5181
|
+
}
|
|
5182
|
+
}
|
|
5183
|
+
if (summary.turnServers && scratch) {
|
|
5184
|
+
summary.turnServers.clientsRelayed = scratch.turnClientIds.size;
|
|
5185
|
+
}
|
|
5186
|
+
}
|
|
5187
|
+
/** Drop every bus subscription. Called when the observer closes. */
|
|
5188
|
+
close() {
|
|
5189
|
+
if (this._closed) return;
|
|
5190
|
+
this._closed = true;
|
|
5191
|
+
for (const { event, listener } of this._listeners) {
|
|
5192
|
+
this._observer.off(event, listener);
|
|
5193
|
+
}
|
|
5194
|
+
this._listeners.length = 0;
|
|
5195
|
+
}
|
|
5196
|
+
/**
|
|
5197
|
+
* Subscribe `listener` to `event`, routed to the summary of the call the event names.
|
|
5198
|
+
*
|
|
5199
|
+
* The `observedCall` is read off the payload rather than closed over, which is what lets one
|
|
5200
|
+
* subscription serve every call.
|
|
5201
|
+
*/
|
|
5202
|
+
_on(event, handler) {
|
|
5203
|
+
const listener = (...args) => {
|
|
5204
|
+
const summary = args[0].observedCall.summary;
|
|
5205
|
+
if (!summary) return;
|
|
5206
|
+
const scratch = this._scratch.get(summary);
|
|
5207
|
+
if (!scratch) return;
|
|
5208
|
+
try {
|
|
5209
|
+
handler(summary, scratch, ...args);
|
|
5210
|
+
} catch (err) {
|
|
5211
|
+
logger8.warn("A call-summary handler for %s threw; continuing. %o", event, err);
|
|
5212
|
+
}
|
|
5213
|
+
};
|
|
5214
|
+
this._observer.on(event, listener);
|
|
5215
|
+
this._listeners.push({ event, listener });
|
|
5216
|
+
}
|
|
5217
|
+
_subscribeBuiltIns() {
|
|
5218
|
+
const include = this._config.include;
|
|
5219
|
+
if (include.includes("clients")) {
|
|
5220
|
+
this._on("client-added", (summary, _scratch, { observedCall, observedClient }) => {
|
|
5221
|
+
const clients = summary.clients;
|
|
5222
|
+
if (!clients) return;
|
|
5223
|
+
clients.joined += 1;
|
|
5224
|
+
clients.peak = Math.max(clients.peak, observedCall.observedClients.size);
|
|
5225
|
+
if (clients.clientIds.length < this._config.maxClientIds) {
|
|
5226
|
+
clients.clientIds.push(observedClient.clientId);
|
|
5227
|
+
} else {
|
|
5228
|
+
summary.truncated = { ...summary.truncated, clientIds: (summary.truncated?.clientIds ?? 0) + 1 };
|
|
5229
|
+
}
|
|
5230
|
+
});
|
|
5231
|
+
this._on("client-closed", (summary) => {
|
|
5232
|
+
if (summary.clients) summary.clients.left += 1;
|
|
5233
|
+
});
|
|
5234
|
+
}
|
|
5235
|
+
if (include.includes("issues")) {
|
|
5236
|
+
this._on("call-issue", (summary, _scratch, { issue }) => {
|
|
5237
|
+
const issues = summary.issues;
|
|
5238
|
+
if (!issues) return;
|
|
5239
|
+
if (issues.length < this._config.maxIssues) issues.push(issue);
|
|
5240
|
+
else summary.truncated = { ...summary.truncated, issues: (summary.truncated?.issues ?? 0) + 1 };
|
|
5241
|
+
});
|
|
5242
|
+
}
|
|
5243
|
+
if (include.includes("scores") || include.includes("turnServers")) {
|
|
5244
|
+
this._on("call-updated", (summary, scratch, { observedCall }) => {
|
|
5245
|
+
if (summary.scores && observedCall.score !== void 0) scratch.scores.push(observedCall.score);
|
|
5246
|
+
if (!summary.turnServers) return;
|
|
5247
|
+
for (const clientId of observedCall.clientsUsedTurn) scratch.turnClientIds.add(clientId);
|
|
5248
|
+
for (const client of observedCall.observedClients.values()) {
|
|
5249
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
5250
|
+
for (const pair of peerConnection.selectedIceCandiadtePairForTurn) {
|
|
5251
|
+
const url = pair.getLocalCandidate()?.url;
|
|
5252
|
+
if (url && !summary.turnServers.serverUrls.includes(url)) {
|
|
5253
|
+
summary.turnServers.serverUrls.push(url);
|
|
5254
|
+
}
|
|
5255
|
+
}
|
|
5256
|
+
}
|
|
5257
|
+
}
|
|
5258
|
+
});
|
|
5259
|
+
}
|
|
5260
|
+
}
|
|
5261
|
+
_subscribeEnrichers() {
|
|
5262
|
+
const enrich = this._config.enrich;
|
|
5263
|
+
if (!enrich) return;
|
|
5264
|
+
for (const name of Object.keys(enrich)) {
|
|
5265
|
+
const enricher = enrich[name];
|
|
5266
|
+
if (!enricher) continue;
|
|
5267
|
+
this._on(name, (summary, _scratch, ...args) => enricher(summary, ...args));
|
|
5268
|
+
}
|
|
5269
|
+
}
|
|
5270
|
+
};
|
|
5271
|
+
|
|
5091
5272
|
// src/detectors/SfuCongestionDetector.ts
|
|
5092
5273
|
var SfuCongestionDetector = class _SfuCongestionDetector {
|
|
5093
5274
|
constructor(_observer, config = {}) {
|
|
@@ -5216,13 +5397,10 @@ var SfuCongestionDetector = class _SfuCongestionDetector {
|
|
|
5216
5397
|
absoluteIncrease: evaluation.absoluteIncrease,
|
|
5217
5398
|
relativeIncrease: evaluation.relativeIncrease
|
|
5218
5399
|
};
|
|
5219
|
-
this._observer.
|
|
5220
|
-
|
|
5221
|
-
|
|
5222
|
-
|
|
5223
|
-
payload
|
|
5224
|
-
},
|
|
5225
|
-
observer: this._observer
|
|
5400
|
+
this._observer.addIssue({
|
|
5401
|
+
type: this._config.emittedObserverIssueType,
|
|
5402
|
+
timestamp: Date.now(),
|
|
5403
|
+
payload: { ...payload }
|
|
5226
5404
|
});
|
|
5227
5405
|
}
|
|
5228
5406
|
/**
|
|
@@ -5336,11 +5514,9 @@ var ObserverConcurrentIssueDetector = class _ObserverConcurrentIssueDetector {
|
|
|
5336
5514
|
this._observer.addIssue({
|
|
5337
5515
|
type: issueType,
|
|
5338
5516
|
timestamp: now,
|
|
5517
|
+
conclusion,
|
|
5339
5518
|
payload: {
|
|
5340
|
-
type: issueType,
|
|
5341
5519
|
issueType: type,
|
|
5342
|
-
scope: "observer",
|
|
5343
|
-
conclusion,
|
|
5344
5520
|
clients: group.totalClients,
|
|
5345
5521
|
affectedClients: group.clientIds.length,
|
|
5346
5522
|
affectedRatio: group.affectedRatio,
|
|
@@ -5513,16 +5689,13 @@ var ClientPopulationIssueDetector = class _ClientPopulationIssueDetector {
|
|
|
5513
5689
|
this._observer.addIssue({
|
|
5514
5690
|
type: ClientPopulationIssueTypes.clientPopulationIssue,
|
|
5515
5691
|
timestamp: now,
|
|
5516
|
-
|
|
5517
|
-
|
|
5518
|
-
|
|
5519
|
-
|
|
5520
|
-
|
|
5521
|
-
|
|
5522
|
-
|
|
5523
|
-
confidence: this._confidenceOf(rollup)
|
|
5524
|
-
}
|
|
5525
|
-
}
|
|
5692
|
+
conclusion: {
|
|
5693
|
+
faultDomain: "client-population",
|
|
5694
|
+
summary: `'${issueType}' is ${this._riskText(rollup.relativeRisk)} more likely on ${population} than on the rest of the fleet (${rollup.affectedClients}/${rollup.clients} vs ${rollup.controlAffectedClients}/${rollup.controlClients})`,
|
|
5695
|
+
recommendation: "this is not an SFU symptom \u2014 look at what those clients share: a recent release, a browser version, or shared/virtualised hardware",
|
|
5696
|
+
confidence: this._confidenceOf(rollup)
|
|
5697
|
+
},
|
|
5698
|
+
payload: { ...rollup }
|
|
5526
5699
|
});
|
|
5527
5700
|
}
|
|
5528
5701
|
}
|
|
@@ -5624,7 +5797,7 @@ var TurnServerHealthDetector = class _TurnServerHealthDetector {
|
|
|
5624
5797
|
this._observer.addIssue({
|
|
5625
5798
|
type: TurnServerHealthTypes.turnServerDegraded,
|
|
5626
5799
|
timestamp: now,
|
|
5627
|
-
payload: {
|
|
5800
|
+
payload: { ...health, otherServers }
|
|
5628
5801
|
});
|
|
5629
5802
|
}
|
|
5630
5803
|
for (const serverUrl of [...this._streaks.keys()]) {
|
|
@@ -5732,7 +5905,6 @@ var TurnServerOutageDetector = class _TurnServerOutageDetector {
|
|
|
5732
5905
|
type: TurnServerOutageTypes.turnServerOutage,
|
|
5733
5906
|
timestamp: now,
|
|
5734
5907
|
payload: {
|
|
5735
|
-
type: TurnServerOutageTypes.turnServerOutage,
|
|
5736
5908
|
serverUrl,
|
|
5737
5909
|
peakClients: peak,
|
|
5738
5910
|
currentClients: live,
|
|
@@ -6006,17 +6178,13 @@ var SimulcastReceiverValidator = class _SimulcastReceiverValidator {
|
|
|
6006
6178
|
this._observer.addIssue({
|
|
6007
6179
|
type: LOWEST_COMMON_DENOMINATOR_ISSUE,
|
|
6008
6180
|
timestamp: decidedAt,
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
recommendation: "check that simulcast/SVC is enabled and layers are chosen per consumer, and that the SFU terminates receiver reports instead of forwarding them; this is a build/config property, not a transient",
|
|
6017
|
-
confidence: 0.8
|
|
6018
|
-
}
|
|
6019
|
-
}
|
|
6181
|
+
conclusion: {
|
|
6182
|
+
faultDomain: "infrastructure",
|
|
6183
|
+
summary: "one bad receiver is dragging a publisher's bitrate down for everyone \u2014 the SFU is not selecting layers per consumer",
|
|
6184
|
+
recommendation: "check that simulcast/SVC is enabled and layers are chosen per consumer, and that the SFU terminates receiver reports instead of forwarding them; this is a build/config property, not a transient",
|
|
6185
|
+
confidence: 0.8
|
|
6186
|
+
},
|
|
6187
|
+
payload: { checks: this._checks, ...outcome.evidence }
|
|
6020
6188
|
});
|
|
6021
6189
|
}
|
|
6022
6190
|
this.onDone(this.report);
|
|
@@ -6119,17 +6287,13 @@ var RemoteTrackResolverValidator = class _RemoteTrackResolverValidator {
|
|
|
6119
6287
|
this._observer.addIssue({
|
|
6120
6288
|
type: UNRESOLVED_TRACK_LINKS_ISSUE,
|
|
6121
6289
|
timestamp: decidedAt,
|
|
6122
|
-
|
|
6123
|
-
|
|
6124
|
-
|
|
6125
|
-
|
|
6126
|
-
|
|
6127
|
-
|
|
6128
|
-
|
|
6129
|
-
recommendation: "check the id the resolver joins on (mediasoup producerId, or your own convention) \u2014 until it links, IssueFanOutDetector, TrackDeliveryMismatchDetector, UnconsumedTrackDetector and SimulcastReceiverValidator all silently do nothing",
|
|
6130
|
-
confidence: 0.9
|
|
6131
|
-
}
|
|
6132
|
-
}
|
|
6290
|
+
conclusion: {
|
|
6291
|
+
faultDomain: "infrastructure",
|
|
6292
|
+
summary: "a RemoteTrackResolver is configured but has never linked a subscriber to a publisher",
|
|
6293
|
+
recommendation: "check the id the resolver joins on (mediasoup producerId, or your own convention) \u2014 until it links, IssueFanOutDetector, TrackDeliveryMismatchDetector, UnconsumedTrackDetector and SimulcastReceiverValidator all silently do nothing",
|
|
6294
|
+
confidence: 0.9
|
|
6295
|
+
},
|
|
6296
|
+
payload: { checks: this._checks, ...outcome.evidence }
|
|
6133
6297
|
});
|
|
6134
6298
|
}
|
|
6135
6299
|
this.onDone(this.report);
|
|
@@ -6240,17 +6404,16 @@ var CodecConsistencyValidator = class _CodecConsistencyValidator {
|
|
|
6240
6404
|
this._observer.addIssue({
|
|
6241
6405
|
type: CODEC_MISMATCH_ISSUE,
|
|
6242
6406
|
timestamp: decidedAt,
|
|
6407
|
+
conclusion: {
|
|
6408
|
+
faultDomain: "infrastructure",
|
|
6409
|
+
summary: outcome.verdict === "codec-split" ? "participants of one call are using different codecs \u2014 an SFU that forwards without transcoding cannot serve all of them" : "the deployment is consistently negotiating a codec other than the expected one",
|
|
6410
|
+
recommendation: outcome.verdict === "codec-split" ? "check codec preferences and any SDP munging; a split usually means one endpoint could not negotiate the preferred codec and the others were not renegotiated with it" : "check codec preferences and endpoint support \u2014 a silent fallback keeps working, at a higher bitrate than you budgeted for",
|
|
6411
|
+
confidence: 0.85
|
|
6412
|
+
},
|
|
6243
6413
|
payload: {
|
|
6244
|
-
type: CODEC_MISMATCH_ISSUE,
|
|
6245
6414
|
verdict: outcome.verdict,
|
|
6246
6415
|
checks: this._checks,
|
|
6247
|
-
evidence: outcome.evidence
|
|
6248
|
-
conclusion: {
|
|
6249
|
-
faultDomain: "infrastructure",
|
|
6250
|
-
summary: outcome.verdict === "codec-split" ? "participants of one call are using different codecs \u2014 an SFU that forwards without transcoding cannot serve all of them" : "the deployment is consistently negotiating a codec other than the expected one",
|
|
6251
|
-
recommendation: outcome.verdict === "codec-split" ? "check codec preferences and any SDP munging; a split usually means one endpoint could not negotiate the preferred codec and the others were not renegotiated with it" : "check codec preferences and endpoint support \u2014 a silent fallback keeps working, at a higher bitrate than you budgeted for",
|
|
6252
|
-
confidence: 0.85
|
|
6253
|
-
}
|
|
6416
|
+
evidence: outcome.evidence
|
|
6254
6417
|
}
|
|
6255
6418
|
});
|
|
6256
6419
|
}
|
|
@@ -6264,7 +6427,7 @@ function mediaKindOf(mimeType) {
|
|
|
6264
6427
|
}
|
|
6265
6428
|
|
|
6266
6429
|
// src/Observer.ts
|
|
6267
|
-
var
|
|
6430
|
+
var logger9 = createLogger("Observer");
|
|
6268
6431
|
var Observer = class extends EventEmitter6 {
|
|
6269
6432
|
observedTURN = new ObservedTURN();
|
|
6270
6433
|
observedCalls = /* @__PURE__ */ new Map();
|
|
@@ -6320,6 +6483,14 @@ var Observer = class extends EventEmitter6 {
|
|
|
6320
6483
|
* ```
|
|
6321
6484
|
*/
|
|
6322
6485
|
callDetectorConfigs = /* @__PURE__ */ new Map();
|
|
6486
|
+
/**
|
|
6487
|
+
* Owns every call's summary: the resolved `config.callSummary`, the bus subscriptions that keep
|
|
6488
|
+
* the summaries current (one per event type, not one per call), and the summaries themselves.
|
|
6489
|
+
*
|
|
6490
|
+
* `undefined` when `config.callSummary` was absent or `null` — so its presence *is* the answer to
|
|
6491
|
+
* "are summaries on", and nothing is subscribed to anything.
|
|
6492
|
+
*/
|
|
6493
|
+
callSummaryCollector;
|
|
6323
6494
|
constructor(config = {}) {
|
|
6324
6495
|
super();
|
|
6325
6496
|
this.setMaxListeners(Infinity);
|
|
@@ -6331,6 +6502,12 @@ var Observer = class extends EventEmitter6 {
|
|
|
6331
6502
|
closeClientIfIdleForMs: 6e4,
|
|
6332
6503
|
...config
|
|
6333
6504
|
};
|
|
6505
|
+
if (this.config.callSummary) {
|
|
6506
|
+
this.callSummaryCollector = new CallSummaryCollector(this, {
|
|
6507
|
+
...defaultCallSummaryConfig,
|
|
6508
|
+
...this.config.callSummary
|
|
6509
|
+
});
|
|
6510
|
+
}
|
|
6334
6511
|
}
|
|
6335
6512
|
get numberOfCalls() {
|
|
6336
6513
|
return this.observedCalls.size;
|
|
@@ -6369,7 +6546,7 @@ var Observer = class extends EventEmitter6 {
|
|
|
6369
6546
|
break;
|
|
6370
6547
|
}
|
|
6371
6548
|
default: {
|
|
6372
|
-
|
|
6549
|
+
logger9.warn("Unknown detector name %s; skipping", name);
|
|
6373
6550
|
return this;
|
|
6374
6551
|
}
|
|
6375
6552
|
}
|
|
@@ -6454,7 +6631,7 @@ var Observer = class extends EventEmitter6 {
|
|
|
6454
6631
|
break;
|
|
6455
6632
|
}
|
|
6456
6633
|
if (!validator) {
|
|
6457
|
-
|
|
6634
|
+
logger9.warn("Unknown validator name %s; skipping", name);
|
|
6458
6635
|
return this;
|
|
6459
6636
|
}
|
|
6460
6637
|
this.validators.add(validator);
|
|
@@ -6485,7 +6662,7 @@ var Observer = class extends EventEmitter6 {
|
|
|
6485
6662
|
try {
|
|
6486
6663
|
validator.cancel(reason);
|
|
6487
6664
|
} catch (err) {
|
|
6488
|
-
|
|
6665
|
+
logger9.warn("Error cancelling validator %s: %o", validator.name, err);
|
|
6489
6666
|
}
|
|
6490
6667
|
}
|
|
6491
6668
|
return matching.length;
|
|
@@ -6494,19 +6671,24 @@ var Observer = class extends EventEmitter6 {
|
|
|
6494
6671
|
if (this.closed || !this.observedCalls.has(callId)) return;
|
|
6495
6672
|
return this.observedCalls.get(callId);
|
|
6496
6673
|
}
|
|
6497
|
-
|
|
6674
|
+
/**
|
|
6675
|
+
* @param acceptCtx the `accept()` context, when this call is being created to receive a sample.
|
|
6676
|
+
* Passed on to `ObserverConfig.createCallAppData`, so the factory can read whatever the caller (or
|
|
6677
|
+
* an accept middleware) put there — a tenant, a region, a trace id.
|
|
6678
|
+
*/
|
|
6679
|
+
createObservedCall(settings, acceptCtx) {
|
|
6498
6680
|
if (this.closed) {
|
|
6499
|
-
|
|
6681
|
+
logger9.warn("Attempted to create a call (callId: %s) on a closed observer", settings.callId);
|
|
6500
6682
|
return void 0;
|
|
6501
6683
|
}
|
|
6502
6684
|
if (this.observedCalls.has(settings.callId)) {
|
|
6503
|
-
|
|
6685
|
+
logger9.warn("Observed Call with id %s already exists; returning the existing instance", settings.callId);
|
|
6504
6686
|
return this.observedCalls.get(settings.callId);
|
|
6505
6687
|
}
|
|
6506
6688
|
const callSettings = {
|
|
6507
6689
|
...settings,
|
|
6508
6690
|
closeCallIfEmptyForMs: settings.closeCallIfEmptyForMs ?? this.config.closeCallIfEmptyForMs,
|
|
6509
|
-
appData: settings.appData ?? this.config.createCallAppData?.({ callId: settings.callId, observer: this })
|
|
6691
|
+
appData: settings.appData ?? this.config.createCallAppData?.({ callId: settings.callId, observer: this, acceptCtx })
|
|
6510
6692
|
};
|
|
6511
6693
|
const callActiveIssuesRegistry = new ActiveIssuesRegistry(this.activeIssuesRegistry);
|
|
6512
6694
|
const observedCall = new ObservedCall(
|
|
@@ -6515,6 +6697,7 @@ var Observer = class extends EventEmitter6 {
|
|
|
6515
6697
|
callActiveIssuesRegistry
|
|
6516
6698
|
);
|
|
6517
6699
|
observedCall.remoteTrackResolver = this.config.createRemoteTrackResolver?.(observedCall);
|
|
6700
|
+
observedCall.enableSummary();
|
|
6518
6701
|
for (const [name, detectorConfig] of this.callDetectorConfigs) {
|
|
6519
6702
|
observedCall.addDetector(name, detectorConfig);
|
|
6520
6703
|
}
|
|
@@ -6532,16 +6715,16 @@ var Observer = class extends EventEmitter6 {
|
|
|
6532
6715
|
this._notify("call-added", { ...this.eventScope, observedCall });
|
|
6533
6716
|
return observedCall;
|
|
6534
6717
|
}
|
|
6535
|
-
getOrCreateObservedCall(settings) {
|
|
6536
|
-
return this.getObservedCall(settings.callId) ?? this.createObservedCall(settings);
|
|
6718
|
+
getOrCreateObservedCall(settings, acceptCtx) {
|
|
6719
|
+
return this.getObservedCall(settings.callId) ?? this.createObservedCall(settings, acceptCtx);
|
|
6537
6720
|
}
|
|
6538
6721
|
createObservedMediasoupRouter(settings) {
|
|
6539
6722
|
if (this.closed) {
|
|
6540
|
-
|
|
6723
|
+
logger9.warn("Attempted to create mediasoup router (id: %d) on a closed observer", settings.router.id);
|
|
6541
6724
|
return void 0;
|
|
6542
6725
|
}
|
|
6543
6726
|
if (this.observedMediasoupRouters.has(settings.router.id)) {
|
|
6544
|
-
|
|
6727
|
+
logger9.warn("Observed Mediasoup Router (id %s) already exists; returning the existing instance", settings.router.id);
|
|
6545
6728
|
return this.observedMediasoupRouters.get(settings.router.id);
|
|
6546
6729
|
}
|
|
6547
6730
|
const observedMediasoupRouter = new ObservedMediasoupRouter(settings);
|
|
@@ -6572,13 +6755,14 @@ var Observer = class extends EventEmitter6 {
|
|
|
6572
6755
|
}
|
|
6573
6756
|
close() {
|
|
6574
6757
|
if (this.closed) {
|
|
6575
|
-
return
|
|
6758
|
+
return logger9.debug("Attempted to close twice");
|
|
6576
6759
|
}
|
|
6577
6760
|
this.closed = true;
|
|
6578
6761
|
for (const call of [...this.observedCalls.values()]) call.close();
|
|
6579
6762
|
this.detectors.clear();
|
|
6580
6763
|
for (const validator of [...this.validators]) validator.cancel("observer closed");
|
|
6581
6764
|
this.validators.clear();
|
|
6765
|
+
this.callSummaryCollector?.close();
|
|
6582
6766
|
this.activeIssuesRegistry.clear();
|
|
6583
6767
|
this._notify("observer-closed", { ...this.eventScope });
|
|
6584
6768
|
}
|
|
@@ -6589,7 +6773,7 @@ var Observer = class extends EventEmitter6 {
|
|
|
6589
6773
|
try {
|
|
6590
6774
|
this.acceptMiddlewares.process({ sample, context });
|
|
6591
6775
|
} catch (err) {
|
|
6592
|
-
|
|
6776
|
+
logger9.warn("An accept middleware threw; dropping the sample. %o", err);
|
|
6593
6777
|
}
|
|
6594
6778
|
if (!sample.callId) {
|
|
6595
6779
|
this._notify("sample-rejected", { ...this.eventScope, reason: "missing-callId", sample });
|
|
@@ -6599,21 +6783,9 @@ var Observer = class extends EventEmitter6 {
|
|
|
6599
6783
|
this._notify("sample-rejected", { ...this.eventScope, reason: "missing-clientId", sample });
|
|
6600
6784
|
return;
|
|
6601
6785
|
}
|
|
6602
|
-
|
|
6603
|
-
if (!call) {
|
|
6604
|
-
call = this.createObservedCall({
|
|
6605
|
-
callId: sample.callId,
|
|
6606
|
-
appData: this.config.createCallAppData?.({ callId: sample.callId, observer: this })
|
|
6607
|
-
});
|
|
6608
|
-
}
|
|
6786
|
+
const call = this.getOrCreateObservedCall({ callId: sample.callId }, context);
|
|
6609
6787
|
if (!call) return;
|
|
6610
|
-
|
|
6611
|
-
if (!client) {
|
|
6612
|
-
client = call.createObservedClient({
|
|
6613
|
-
clientId: sample.clientId,
|
|
6614
|
-
appData: this.config.createClientAppData?.({ clientId: sample.clientId, observedCall: call })
|
|
6615
|
-
});
|
|
6616
|
-
}
|
|
6788
|
+
const client = call.getOrCreateObservedClient({ clientId: sample.clientId }, context);
|
|
6617
6789
|
if (!client) return;
|
|
6618
6790
|
client.accept(sample, context);
|
|
6619
6791
|
}
|
|
@@ -6641,7 +6813,7 @@ var Observer = class extends EventEmitter6 {
|
|
|
6641
6813
|
try {
|
|
6642
6814
|
validator.update();
|
|
6643
6815
|
} catch (err) {
|
|
6644
|
-
|
|
6816
|
+
logger9.warn("Error running validator %s: %o", validator.name, err);
|
|
6645
6817
|
}
|
|
6646
6818
|
}
|
|
6647
6819
|
this._notify("observer-updated", { ...this.eventScope });
|
|
@@ -6654,7 +6826,7 @@ var Observer = class extends EventEmitter6 {
|
|
|
6654
6826
|
*/
|
|
6655
6827
|
addIssue(issue) {
|
|
6656
6828
|
if (this.closed) return;
|
|
6657
|
-
this._notify("observer-issue", { ...this.eventScope, issue });
|
|
6829
|
+
this._notify("observer-issue", { ...this.eventScope, issue: { ...issue, scope: "observer" } });
|
|
6658
6830
|
}
|
|
6659
6831
|
/** Emit an Observer-bus event. */
|
|
6660
6832
|
_notify(type, ...args) {
|
|
@@ -6772,18 +6944,11 @@ var CallHealthAggregator = class {
|
|
|
6772
6944
|
}
|
|
6773
6945
|
};
|
|
6774
6946
|
|
|
6775
|
-
// src/common/
|
|
6776
|
-
function issuePayloadOf(issue) {
|
|
6777
|
-
const payload = issue.payload;
|
|
6778
|
-
if (payload === void 0) return void 0;
|
|
6779
|
-
return typeof payload === "string" ? parseJsonObject(payload) : payload;
|
|
6780
|
-
}
|
|
6947
|
+
// src/common/Issue.ts
|
|
6781
6948
|
function issuePayloadAsString(issue) {
|
|
6782
|
-
|
|
6783
|
-
if (payload === void 0) return void 0;
|
|
6784
|
-
if (typeof payload === "string") return payload;
|
|
6949
|
+
if (issue.payload === void 0) return void 0;
|
|
6785
6950
|
try {
|
|
6786
|
-
return JSON.stringify(payload);
|
|
6951
|
+
return JSON.stringify(issue.payload);
|
|
6787
6952
|
} catch {
|
|
6788
6953
|
return void 0;
|
|
6789
6954
|
}
|
|
@@ -7077,6 +7242,7 @@ export {
|
|
|
7077
7242
|
CallConcurrentIssueDetector,
|
|
7078
7243
|
CallConcurrentIssueTypes,
|
|
7079
7244
|
CallHealthAggregator,
|
|
7245
|
+
CallSummaryCollector,
|
|
7080
7246
|
ClientEventTypes,
|
|
7081
7247
|
ClientMetaTypes,
|
|
7082
7248
|
ClientPopulationIssueDetector,
|
|
@@ -7135,16 +7301,17 @@ export {
|
|
|
7135
7301
|
concludeObserverIssue,
|
|
7136
7302
|
correlation,
|
|
7137
7303
|
counterDelta,
|
|
7304
|
+
createCallSummary,
|
|
7138
7305
|
createDefaultMediasoupRemoteTrackResolverFactory,
|
|
7139
7306
|
createInMemorySink,
|
|
7140
7307
|
createJsonlFileSink,
|
|
7141
7308
|
createJsonlFileSinkFactory,
|
|
7142
7309
|
createLogger,
|
|
7143
7310
|
createP2pRemoteTrackResolverFactory,
|
|
7311
|
+
defaultCallSummaryConfig,
|
|
7144
7312
|
defaultClientHealthThresholds,
|
|
7145
7313
|
isClientIssueResolutionEntry,
|
|
7146
7314
|
issuePayloadAsString,
|
|
7147
|
-
issuePayloadOf,
|
|
7148
7315
|
mannKendall,
|
|
7149
7316
|
mannKendallVerdict,
|
|
7150
7317
|
median,
|