@observertc/observer-js 1.0.0-beta.16 → 1.0.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -34
- package/dist/index.d.mts +440 -139
- package/dist/index.d.ts +440 -139
- package/dist/index.js +298 -129
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +295 -128
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -35,6 +35,7 @@ __export(src_exports, {
|
|
|
35
35
|
CallConcurrentIssueDetector: () => CallConcurrentIssueDetector,
|
|
36
36
|
CallConcurrentIssueTypes: () => CallConcurrentIssueTypes,
|
|
37
37
|
CallHealthAggregator: () => CallHealthAggregator,
|
|
38
|
+
CallSummaryCollector: () => CallSummaryCollector,
|
|
38
39
|
ClientEventTypes: () => ClientEventTypes,
|
|
39
40
|
ClientMetaTypes: () => ClientMetaTypes,
|
|
40
41
|
ClientPopulationIssueDetector: () => ClientPopulationIssueDetector,
|
|
@@ -93,16 +94,17 @@ __export(src_exports, {
|
|
|
93
94
|
concludeObserverIssue: () => concludeObserverIssue,
|
|
94
95
|
correlation: () => correlation,
|
|
95
96
|
counterDelta: () => counterDelta,
|
|
97
|
+
createCallSummary: () => createCallSummary,
|
|
96
98
|
createDefaultMediasoupRemoteTrackResolverFactory: () => createDefaultMediasoupRemoteTrackResolverFactory,
|
|
97
99
|
createInMemorySink: () => createInMemorySink,
|
|
98
100
|
createJsonlFileSink: () => createJsonlFileSink,
|
|
99
101
|
createJsonlFileSinkFactory: () => createJsonlFileSinkFactory,
|
|
100
102
|
createLogger: () => createLogger,
|
|
101
103
|
createP2pRemoteTrackResolverFactory: () => createP2pRemoteTrackResolverFactory,
|
|
104
|
+
defaultCallSummaryConfig: () => defaultCallSummaryConfig,
|
|
102
105
|
defaultClientHealthThresholds: () => defaultClientHealthThresholds,
|
|
103
106
|
isClientIssueResolutionEntry: () => isClientIssueResolutionEntry,
|
|
104
107
|
issuePayloadAsString: () => issuePayloadAsString,
|
|
105
|
-
issuePayloadOf: () => issuePayloadOf,
|
|
106
108
|
mannKendall: () => mannKendall,
|
|
107
109
|
mannKendallVerdict: () => mannKendallVerdict,
|
|
108
110
|
median: () => median,
|
|
@@ -155,8 +157,8 @@ function createLogger(moduleName) {
|
|
|
155
157
|
}
|
|
156
158
|
}();
|
|
157
159
|
}
|
|
158
|
-
function setObserverLogger(
|
|
159
|
-
mainLogger =
|
|
160
|
+
function setObserverLogger(logger10) {
|
|
161
|
+
mainLogger = logger10;
|
|
160
162
|
}
|
|
161
163
|
|
|
162
164
|
// src/ObservedCall.ts
|
|
@@ -3574,7 +3576,6 @@ var UnconsumedTrackDetector = class _UnconsumedTrackDetector {
|
|
|
3574
3576
|
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
3575
3577
|
timestamp: now,
|
|
3576
3578
|
payload: {
|
|
3577
|
-
type: UnconsumedTrackTypes.unconsumedPublishedTrack,
|
|
3578
3579
|
trackId: outboundTrack.id,
|
|
3579
3580
|
kind: outboundTrack.kind,
|
|
3580
3581
|
publisherClientId: peerConnection?.client.clientId,
|
|
@@ -3720,7 +3721,6 @@ var TrackDeliveryMismatchDetector = class _TrackDeliveryMismatchDetector {
|
|
|
3720
3721
|
type,
|
|
3721
3722
|
timestamp: now,
|
|
3722
3723
|
payload: {
|
|
3723
|
-
type,
|
|
3724
3724
|
trackId: outboundTrackId,
|
|
3725
3725
|
publisherClientId: delivery.publisherClientId,
|
|
3726
3726
|
publisherSending: delivery.publisherSending,
|
|
@@ -3973,12 +3973,9 @@ var CallConcurrentIssueDetector = class _CallConcurrentIssueDetector {
|
|
|
3973
3973
|
this._call.addIssue({
|
|
3974
3974
|
type: issueType,
|
|
3975
3975
|
timestamp: now,
|
|
3976
|
+
conclusion,
|
|
3976
3977
|
payload: {
|
|
3977
|
-
type: issueType,
|
|
3978
3978
|
issueType: type,
|
|
3979
|
-
scope: "call",
|
|
3980
|
-
conclusion,
|
|
3981
|
-
callId: this._call.callId,
|
|
3982
3979
|
clients: group.totalClients,
|
|
3983
3980
|
affectedClients: group.clientIds.length,
|
|
3984
3981
|
affectedRatio: group.affectedRatio,
|
|
@@ -4102,10 +4099,9 @@ var IssueFanOutDetector = class _IssueFanOutDetector {
|
|
|
4102
4099
|
this._call.addIssue({
|
|
4103
4100
|
type,
|
|
4104
4101
|
timestamp: now,
|
|
4102
|
+
conclusion,
|
|
4105
4103
|
payload: {
|
|
4106
|
-
type,
|
|
4107
4104
|
issueType,
|
|
4108
|
-
conclusion,
|
|
4109
4105
|
trackId: publisher.id,
|
|
4110
4106
|
kind: publisher.kind,
|
|
4111
4107
|
publisherClientId: publisher.getPeerConnection().client.clientId,
|
|
@@ -4252,18 +4248,14 @@ var PublisherFaultCorroborationDetector = class _PublisherFaultCorroborationDete
|
|
|
4252
4248
|
this._call.addIssue({
|
|
4253
4249
|
type: PublisherFaultTypes.corroboratedPublisherFault,
|
|
4254
4250
|
timestamp: now,
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
|
|
4263
|
-
// Higher than any single-ended finding: two independent parties, one conclusion.
|
|
4264
|
-
confidence: 0.9
|
|
4265
|
-
}
|
|
4266
|
-
}
|
|
4251
|
+
conclusion: {
|
|
4252
|
+
faultDomain: "published-track",
|
|
4253
|
+
summary: `${fault.publisherClientId} reports ${fault.publisherIssueTypes.join(", ")} on track ${fault.trackId} while ${fault.affectedReceivers} of ${fault.receivers} subscribers report ${fault.receiverIssueTypes.join(", ")} \u2014 both ends agree`,
|
|
4254
|
+
recommendation: "the source is implicated, not inferred: check that publisher's capture, encoder and uplink before looking at the SFU or the receivers",
|
|
4255
|
+
// Higher than any single-ended finding: two independent parties, one conclusion.
|
|
4256
|
+
confidence: 0.9
|
|
4257
|
+
},
|
|
4258
|
+
payload: { ...fault }
|
|
4267
4259
|
});
|
|
4268
4260
|
}
|
|
4269
4261
|
}
|
|
@@ -4319,6 +4311,14 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
4319
4311
|
value: void 0
|
|
4320
4312
|
};
|
|
4321
4313
|
remoteTrackResolver;
|
|
4314
|
+
/**
|
|
4315
|
+
* The accumulating record of this call's life, or `undefined` when no summary was configured.
|
|
4316
|
+
*
|
|
4317
|
+
* Live — read it at any point during the call. It is also delivered once on `call-summary` when
|
|
4318
|
+
* the call closes. See `CallSummary`: an absent section means "not collected", never "nothing
|
|
4319
|
+
* happened".
|
|
4320
|
+
*/
|
|
4321
|
+
summary;
|
|
4322
4322
|
/**
|
|
4323
4323
|
* Published tracks that currently have **no** subscriber linked to them.
|
|
4324
4324
|
*
|
|
@@ -4396,6 +4396,21 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
4396
4396
|
this.detectors.add(detector);
|
|
4397
4397
|
return this;
|
|
4398
4398
|
}
|
|
4399
|
+
/**
|
|
4400
|
+
* Start accumulating this call's summary, if the observer was configured for summaries.
|
|
4401
|
+
*
|
|
4402
|
+
* Called by `createObservedCall`; you should not need it. It takes no configuration of its own on
|
|
4403
|
+
* purpose: the collector subscribes to exactly the events the observer's `include` requires, so a
|
|
4404
|
+
* per-call section outside that set would be created and then never written to — an empty section
|
|
4405
|
+
* that reads as "nothing happened". One shape per observer is the only shape that can be filled.
|
|
4406
|
+
*
|
|
4407
|
+
* The collector builds it rather than this method, so the resolved configuration never has to
|
|
4408
|
+
* leave the one object that owns it. Returns `undefined` when summaries are off, and is
|
|
4409
|
+
* idempotent: an existing summary is kept, not restarted.
|
|
4410
|
+
*/
|
|
4411
|
+
enableSummary() {
|
|
4412
|
+
return this.summary ??= this.observer.callSummaryCollector?.createSummary(this.callId);
|
|
4413
|
+
}
|
|
4399
4414
|
/**
|
|
4400
4415
|
* Remove a detector from **this call** by name, returning how many were removed.
|
|
4401
4416
|
*
|
|
@@ -4420,12 +4435,13 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
4420
4435
|
/**
|
|
4421
4436
|
* Raise a call-level (server-side) finding; surfaced on the Observer bus as `call-issue`.
|
|
4422
4437
|
*
|
|
4423
|
-
* `payload`
|
|
4424
|
-
* serialise for.
|
|
4438
|
+
* `payload` is an **object** and holds evidence only — it is delivered to an in-process handler,
|
|
4439
|
+
* so there is nothing to serialise for. `scope` is stamped here, and the `callId` is already on
|
|
4440
|
+
* the event, so neither belongs in the payload. Put the interpretation in `conclusion`.
|
|
4425
4441
|
*/
|
|
4426
4442
|
addIssue(issue) {
|
|
4427
4443
|
if (this.closed) return;
|
|
4428
|
-
this._notify("call-issue", { ...this.eventScope, issue });
|
|
4444
|
+
this._notify("call-issue", { ...this.eventScope, issue: { ...issue, scope: "call" } });
|
|
4429
4445
|
}
|
|
4430
4446
|
close() {
|
|
4431
4447
|
if (this.closed) return;
|
|
@@ -4446,6 +4462,10 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
4446
4462
|
if (this.startedAt === void 0) this.startedAt = minSampleTimestamps;
|
|
4447
4463
|
if (this.endedAt === void 0) this.endedAt = maxSampleTimestamps;
|
|
4448
4464
|
this.closedAt = Date.now();
|
|
4465
|
+
if (this.summary) {
|
|
4466
|
+
this.observer.callSummaryCollector?.finalise(this);
|
|
4467
|
+
this._notify("call-summary", { ...this.eventScope, summary: this.summary });
|
|
4468
|
+
}
|
|
4449
4469
|
this.detectors.clear();
|
|
4450
4470
|
this.activeIssuesRegistry.clear();
|
|
4451
4471
|
this.emit("close");
|
|
@@ -4455,7 +4475,7 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
4455
4475
|
if (this.closed || !this.observedClients.has(clientId)) return;
|
|
4456
4476
|
return this.observedClients.get(clientId);
|
|
4457
4477
|
}
|
|
4458
|
-
createObservedClient(settings) {
|
|
4478
|
+
createObservedClient(settings, acceptCtx) {
|
|
4459
4479
|
if (this.closed) {
|
|
4460
4480
|
logger4.warn("Attempted to create a client (clientId: %s) on a closed call %s", settings.clientId, this.callId);
|
|
4461
4481
|
return void 0;
|
|
@@ -4464,15 +4484,18 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
4464
4484
|
logger4.warn("Client with id %s already exists in call %s; returning the existing instance", settings.clientId, this.callId);
|
|
4465
4485
|
return this.observedClients.get(settings.clientId);
|
|
4466
4486
|
}
|
|
4467
|
-
|
|
4468
|
-
settings
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4487
|
+
const clientSettings = {
|
|
4488
|
+
...settings,
|
|
4489
|
+
closeClientIfIdleForMs: settings.closeClientIfIdleForMs || this.observer.config.closeClientIfIdleForMs,
|
|
4490
|
+
appData: settings.appData ?? this.observer.config.createClientAppData?.({
|
|
4491
|
+
clientId: settings.clientId,
|
|
4492
|
+
observedCall: this,
|
|
4493
|
+
acceptCtx
|
|
4494
|
+
})
|
|
4495
|
+
};
|
|
4473
4496
|
const observedClientIssueRegistry = new ObservedClientIssueRegistry(this.activeIssuesRegistry);
|
|
4474
4497
|
const result = new ObservedClient(
|
|
4475
|
-
|
|
4498
|
+
clientSettings,
|
|
4476
4499
|
this,
|
|
4477
4500
|
observedClientIssueRegistry
|
|
4478
4501
|
);
|
|
@@ -4519,8 +4542,8 @@ var ObservedCall = class extends import_events3.EventEmitter {
|
|
|
4519
4542
|
}
|
|
4520
4543
|
return result;
|
|
4521
4544
|
}
|
|
4522
|
-
getOrCreateObservedClient(settings) {
|
|
4523
|
-
return this.getObservedClient(settings.clientId) ?? this.createObservedClient(settings);
|
|
4545
|
+
getOrCreateObservedClient(settings, acceptCtx) {
|
|
4546
|
+
return this.getObservedClient(settings.clientId) ?? this.createObservedClient(settings, acceptCtx);
|
|
4524
4547
|
}
|
|
4525
4548
|
update(context) {
|
|
4526
4549
|
if (this.closed) return;
|
|
@@ -5206,6 +5229,166 @@ var ActiveIssuesRegistry = class {
|
|
|
5206
5229
|
}
|
|
5207
5230
|
};
|
|
5208
5231
|
|
|
5232
|
+
// src/summaries/CallSummary.ts
|
|
5233
|
+
var defaultCallSummaryConfig = {
|
|
5234
|
+
include: [],
|
|
5235
|
+
maxIssues: 500,
|
|
5236
|
+
maxClientIds: 1e4
|
|
5237
|
+
};
|
|
5238
|
+
function createCallSummary(callId, config) {
|
|
5239
|
+
const summary = { callId, attachments: {} };
|
|
5240
|
+
if (config.include.includes("clients")) {
|
|
5241
|
+
summary.clients = { clientIds: [], peak: 0, joined: 0, left: 0 };
|
|
5242
|
+
}
|
|
5243
|
+
if (config.include.includes("issues")) {
|
|
5244
|
+
summary.issues = [];
|
|
5245
|
+
}
|
|
5246
|
+
if (config.include.includes("turnServers")) {
|
|
5247
|
+
summary.turnServers = { serverUrls: [], clientsRelayed: 0 };
|
|
5248
|
+
}
|
|
5249
|
+
if (config.include.includes("scores")) {
|
|
5250
|
+
summary.scores = { samples: 0 };
|
|
5251
|
+
}
|
|
5252
|
+
return summary;
|
|
5253
|
+
}
|
|
5254
|
+
|
|
5255
|
+
// src/summaries/CallSummaryCollector.ts
|
|
5256
|
+
var logger8 = createLogger("CallSummaryCollector");
|
|
5257
|
+
var CallSummaryCollector = class {
|
|
5258
|
+
constructor(_observer, _config) {
|
|
5259
|
+
this._observer = _observer;
|
|
5260
|
+
this._config = _config;
|
|
5261
|
+
this._subscribeBuiltIns();
|
|
5262
|
+
this._subscribeEnrichers();
|
|
5263
|
+
}
|
|
5264
|
+
_observer;
|
|
5265
|
+
_config;
|
|
5266
|
+
_scratch = /* @__PURE__ */ new WeakMap();
|
|
5267
|
+
_listeners = [];
|
|
5268
|
+
_closed = false;
|
|
5269
|
+
/**
|
|
5270
|
+
* Build a summary for `callId` and start tracking it.
|
|
5271
|
+
*
|
|
5272
|
+
* Creating it here, rather than letting the call create one and hand it over, keeps the resolved
|
|
5273
|
+
* configuration inside the single object that owns it — and makes it impossible to end up with a
|
|
5274
|
+
* summary whose sections nobody subscribed to fill.
|
|
5275
|
+
*/
|
|
5276
|
+
createSummary(callId) {
|
|
5277
|
+
const summary = createCallSummary(callId, this._config);
|
|
5278
|
+
this._scratch.set(summary, { scores: [], turnClientIds: /* @__PURE__ */ new Set() });
|
|
5279
|
+
return summary;
|
|
5280
|
+
}
|
|
5281
|
+
/**
|
|
5282
|
+
* Finalise `call`'s summary: fold in what only makes sense once, and stamp the closing times.
|
|
5283
|
+
*
|
|
5284
|
+
* Percentiles are computed here rather than on every update — a median recomputed per tick over a
|
|
5285
|
+
* growing array is quadratic work to produce a number nobody reads until the end.
|
|
5286
|
+
*/
|
|
5287
|
+
finalise(call) {
|
|
5288
|
+
const summary = call.summary;
|
|
5289
|
+
if (!summary) return;
|
|
5290
|
+
const scratch = this._scratch.get(summary);
|
|
5291
|
+
summary.startedAt = call.startedAt;
|
|
5292
|
+
summary.endedAt = call.endedAt;
|
|
5293
|
+
summary.durationInMs = summary.startedAt !== void 0 && summary.endedAt !== void 0 ? Math.max(0, summary.endedAt - summary.startedAt) : void 0;
|
|
5294
|
+
summary.closedAt = Date.now();
|
|
5295
|
+
if (summary.scores && scratch) {
|
|
5296
|
+
summary.scores.samples = scratch.scores.length;
|
|
5297
|
+
if (0 < scratch.scores.length) {
|
|
5298
|
+
summary.scores.min = Math.min(...scratch.scores);
|
|
5299
|
+
summary.scores.max = Math.max(...scratch.scores);
|
|
5300
|
+
summary.scores.median = percentile(scratch.scores, 0.5);
|
|
5301
|
+
}
|
|
5302
|
+
}
|
|
5303
|
+
if (summary.turnServers && scratch) {
|
|
5304
|
+
summary.turnServers.clientsRelayed = scratch.turnClientIds.size;
|
|
5305
|
+
}
|
|
5306
|
+
}
|
|
5307
|
+
/** Drop every bus subscription. Called when the observer closes. */
|
|
5308
|
+
close() {
|
|
5309
|
+
if (this._closed) return;
|
|
5310
|
+
this._closed = true;
|
|
5311
|
+
for (const { event, listener } of this._listeners) {
|
|
5312
|
+
this._observer.off(event, listener);
|
|
5313
|
+
}
|
|
5314
|
+
this._listeners.length = 0;
|
|
5315
|
+
}
|
|
5316
|
+
/**
|
|
5317
|
+
* Subscribe `listener` to `event`, routed to the summary of the call the event names.
|
|
5318
|
+
*
|
|
5319
|
+
* The `observedCall` is read off the payload rather than closed over, which is what lets one
|
|
5320
|
+
* subscription serve every call.
|
|
5321
|
+
*/
|
|
5322
|
+
_on(event, handler) {
|
|
5323
|
+
const listener = (...args) => {
|
|
5324
|
+
const summary = args[0].observedCall.summary;
|
|
5325
|
+
if (!summary) return;
|
|
5326
|
+
const scratch = this._scratch.get(summary);
|
|
5327
|
+
if (!scratch) return;
|
|
5328
|
+
try {
|
|
5329
|
+
handler(summary, scratch, ...args);
|
|
5330
|
+
} catch (err) {
|
|
5331
|
+
logger8.warn("A call-summary handler for %s threw; continuing. %o", event, err);
|
|
5332
|
+
}
|
|
5333
|
+
};
|
|
5334
|
+
this._observer.on(event, listener);
|
|
5335
|
+
this._listeners.push({ event, listener });
|
|
5336
|
+
}
|
|
5337
|
+
_subscribeBuiltIns() {
|
|
5338
|
+
const include = this._config.include;
|
|
5339
|
+
if (include.includes("clients")) {
|
|
5340
|
+
this._on("client-added", (summary, _scratch, { observedCall, observedClient }) => {
|
|
5341
|
+
const clients = summary.clients;
|
|
5342
|
+
if (!clients) return;
|
|
5343
|
+
clients.joined += 1;
|
|
5344
|
+
clients.peak = Math.max(clients.peak, observedCall.observedClients.size);
|
|
5345
|
+
if (clients.clientIds.length < this._config.maxClientIds) {
|
|
5346
|
+
clients.clientIds.push(observedClient.clientId);
|
|
5347
|
+
} else {
|
|
5348
|
+
summary.truncated = { ...summary.truncated, clientIds: (summary.truncated?.clientIds ?? 0) + 1 };
|
|
5349
|
+
}
|
|
5350
|
+
});
|
|
5351
|
+
this._on("client-closed", (summary) => {
|
|
5352
|
+
if (summary.clients) summary.clients.left += 1;
|
|
5353
|
+
});
|
|
5354
|
+
}
|
|
5355
|
+
if (include.includes("issues")) {
|
|
5356
|
+
this._on("call-issue", (summary, _scratch, { issue }) => {
|
|
5357
|
+
const issues = summary.issues;
|
|
5358
|
+
if (!issues) return;
|
|
5359
|
+
if (issues.length < this._config.maxIssues) issues.push(issue);
|
|
5360
|
+
else summary.truncated = { ...summary.truncated, issues: (summary.truncated?.issues ?? 0) + 1 };
|
|
5361
|
+
});
|
|
5362
|
+
}
|
|
5363
|
+
if (include.includes("scores") || include.includes("turnServers")) {
|
|
5364
|
+
this._on("call-updated", (summary, scratch, { observedCall }) => {
|
|
5365
|
+
if (summary.scores && observedCall.score !== void 0) scratch.scores.push(observedCall.score);
|
|
5366
|
+
if (!summary.turnServers) return;
|
|
5367
|
+
for (const clientId of observedCall.clientsUsedTurn) scratch.turnClientIds.add(clientId);
|
|
5368
|
+
for (const client of observedCall.observedClients.values()) {
|
|
5369
|
+
for (const peerConnection of client.observedPeerConnections.values()) {
|
|
5370
|
+
for (const pair of peerConnection.selectedIceCandiadtePairForTurn) {
|
|
5371
|
+
const url = pair.getLocalCandidate()?.url;
|
|
5372
|
+
if (url && !summary.turnServers.serverUrls.includes(url)) {
|
|
5373
|
+
summary.turnServers.serverUrls.push(url);
|
|
5374
|
+
}
|
|
5375
|
+
}
|
|
5376
|
+
}
|
|
5377
|
+
}
|
|
5378
|
+
});
|
|
5379
|
+
}
|
|
5380
|
+
}
|
|
5381
|
+
_subscribeEnrichers() {
|
|
5382
|
+
const enrich = this._config.enrich;
|
|
5383
|
+
if (!enrich) return;
|
|
5384
|
+
for (const name of Object.keys(enrich)) {
|
|
5385
|
+
const enricher = enrich[name];
|
|
5386
|
+
if (!enricher) continue;
|
|
5387
|
+
this._on(name, (summary, _scratch, ...args) => enricher(summary, ...args));
|
|
5388
|
+
}
|
|
5389
|
+
}
|
|
5390
|
+
};
|
|
5391
|
+
|
|
5209
5392
|
// src/detectors/SfuCongestionDetector.ts
|
|
5210
5393
|
var SfuCongestionDetector = class _SfuCongestionDetector {
|
|
5211
5394
|
constructor(_observer, config = {}) {
|
|
@@ -5334,13 +5517,10 @@ var SfuCongestionDetector = class _SfuCongestionDetector {
|
|
|
5334
5517
|
absoluteIncrease: evaluation.absoluteIncrease,
|
|
5335
5518
|
relativeIncrease: evaluation.relativeIncrease
|
|
5336
5519
|
};
|
|
5337
|
-
this._observer.
|
|
5338
|
-
|
|
5339
|
-
|
|
5340
|
-
|
|
5341
|
-
payload
|
|
5342
|
-
},
|
|
5343
|
-
observer: this._observer
|
|
5520
|
+
this._observer.addIssue({
|
|
5521
|
+
type: this._config.emittedObserverIssueType,
|
|
5522
|
+
timestamp: Date.now(),
|
|
5523
|
+
payload: { ...payload }
|
|
5344
5524
|
});
|
|
5345
5525
|
}
|
|
5346
5526
|
/**
|
|
@@ -5454,11 +5634,9 @@ var ObserverConcurrentIssueDetector = class _ObserverConcurrentIssueDetector {
|
|
|
5454
5634
|
this._observer.addIssue({
|
|
5455
5635
|
type: issueType,
|
|
5456
5636
|
timestamp: now,
|
|
5637
|
+
conclusion,
|
|
5457
5638
|
payload: {
|
|
5458
|
-
type: issueType,
|
|
5459
5639
|
issueType: type,
|
|
5460
|
-
scope: "observer",
|
|
5461
|
-
conclusion,
|
|
5462
5640
|
clients: group.totalClients,
|
|
5463
5641
|
affectedClients: group.clientIds.length,
|
|
5464
5642
|
affectedRatio: group.affectedRatio,
|
|
@@ -5631,16 +5809,13 @@ var ClientPopulationIssueDetector = class _ClientPopulationIssueDetector {
|
|
|
5631
5809
|
this._observer.addIssue({
|
|
5632
5810
|
type: ClientPopulationIssueTypes.clientPopulationIssue,
|
|
5633
5811
|
timestamp: now,
|
|
5634
|
-
|
|
5635
|
-
|
|
5636
|
-
|
|
5637
|
-
|
|
5638
|
-
|
|
5639
|
-
|
|
5640
|
-
|
|
5641
|
-
confidence: this._confidenceOf(rollup)
|
|
5642
|
-
}
|
|
5643
|
-
}
|
|
5812
|
+
conclusion: {
|
|
5813
|
+
faultDomain: "client-population",
|
|
5814
|
+
summary: `'${issueType}' is ${this._riskText(rollup.relativeRisk)} more likely on ${population} than on the rest of the fleet (${rollup.affectedClients}/${rollup.clients} vs ${rollup.controlAffectedClients}/${rollup.controlClients})`,
|
|
5815
|
+
recommendation: "this is not an SFU symptom \u2014 look at what those clients share: a recent release, a browser version, or shared/virtualised hardware",
|
|
5816
|
+
confidence: this._confidenceOf(rollup)
|
|
5817
|
+
},
|
|
5818
|
+
payload: { ...rollup }
|
|
5644
5819
|
});
|
|
5645
5820
|
}
|
|
5646
5821
|
}
|
|
@@ -5742,7 +5917,7 @@ var TurnServerHealthDetector = class _TurnServerHealthDetector {
|
|
|
5742
5917
|
this._observer.addIssue({
|
|
5743
5918
|
type: TurnServerHealthTypes.turnServerDegraded,
|
|
5744
5919
|
timestamp: now,
|
|
5745
|
-
payload: {
|
|
5920
|
+
payload: { ...health, otherServers }
|
|
5746
5921
|
});
|
|
5747
5922
|
}
|
|
5748
5923
|
for (const serverUrl of [...this._streaks.keys()]) {
|
|
@@ -5850,7 +6025,6 @@ var TurnServerOutageDetector = class _TurnServerOutageDetector {
|
|
|
5850
6025
|
type: TurnServerOutageTypes.turnServerOutage,
|
|
5851
6026
|
timestamp: now,
|
|
5852
6027
|
payload: {
|
|
5853
|
-
type: TurnServerOutageTypes.turnServerOutage,
|
|
5854
6028
|
serverUrl,
|
|
5855
6029
|
peakClients: peak,
|
|
5856
6030
|
currentClients: live,
|
|
@@ -6124,17 +6298,13 @@ var SimulcastReceiverValidator = class _SimulcastReceiverValidator {
|
|
|
6124
6298
|
this._observer.addIssue({
|
|
6125
6299
|
type: LOWEST_COMMON_DENOMINATOR_ISSUE,
|
|
6126
6300
|
timestamp: decidedAt,
|
|
6127
|
-
|
|
6128
|
-
|
|
6129
|
-
|
|
6130
|
-
|
|
6131
|
-
|
|
6132
|
-
|
|
6133
|
-
|
|
6134
|
-
recommendation: "check that simulcast/SVC is enabled and layers are chosen per consumer, and that the SFU terminates receiver reports instead of forwarding them; this is a build/config property, not a transient",
|
|
6135
|
-
confidence: 0.8
|
|
6136
|
-
}
|
|
6137
|
-
}
|
|
6301
|
+
conclusion: {
|
|
6302
|
+
faultDomain: "infrastructure",
|
|
6303
|
+
summary: "one bad receiver is dragging a publisher's bitrate down for everyone \u2014 the SFU is not selecting layers per consumer",
|
|
6304
|
+
recommendation: "check that simulcast/SVC is enabled and layers are chosen per consumer, and that the SFU terminates receiver reports instead of forwarding them; this is a build/config property, not a transient",
|
|
6305
|
+
confidence: 0.8
|
|
6306
|
+
},
|
|
6307
|
+
payload: { checks: this._checks, ...outcome.evidence }
|
|
6138
6308
|
});
|
|
6139
6309
|
}
|
|
6140
6310
|
this.onDone(this.report);
|
|
@@ -6237,17 +6407,13 @@ var RemoteTrackResolverValidator = class _RemoteTrackResolverValidator {
|
|
|
6237
6407
|
this._observer.addIssue({
|
|
6238
6408
|
type: UNRESOLVED_TRACK_LINKS_ISSUE,
|
|
6239
6409
|
timestamp: decidedAt,
|
|
6240
|
-
|
|
6241
|
-
|
|
6242
|
-
|
|
6243
|
-
|
|
6244
|
-
|
|
6245
|
-
|
|
6246
|
-
|
|
6247
|
-
recommendation: "check the id the resolver joins on (mediasoup producerId, or your own convention) \u2014 until it links, IssueFanOutDetector, TrackDeliveryMismatchDetector, UnconsumedTrackDetector and SimulcastReceiverValidator all silently do nothing",
|
|
6248
|
-
confidence: 0.9
|
|
6249
|
-
}
|
|
6250
|
-
}
|
|
6410
|
+
conclusion: {
|
|
6411
|
+
faultDomain: "infrastructure",
|
|
6412
|
+
summary: "a RemoteTrackResolver is configured but has never linked a subscriber to a publisher",
|
|
6413
|
+
recommendation: "check the id the resolver joins on (mediasoup producerId, or your own convention) \u2014 until it links, IssueFanOutDetector, TrackDeliveryMismatchDetector, UnconsumedTrackDetector and SimulcastReceiverValidator all silently do nothing",
|
|
6414
|
+
confidence: 0.9
|
|
6415
|
+
},
|
|
6416
|
+
payload: { checks: this._checks, ...outcome.evidence }
|
|
6251
6417
|
});
|
|
6252
6418
|
}
|
|
6253
6419
|
this.onDone(this.report);
|
|
@@ -6358,17 +6524,16 @@ var CodecConsistencyValidator = class _CodecConsistencyValidator {
|
|
|
6358
6524
|
this._observer.addIssue({
|
|
6359
6525
|
type: CODEC_MISMATCH_ISSUE,
|
|
6360
6526
|
timestamp: decidedAt,
|
|
6527
|
+
conclusion: {
|
|
6528
|
+
faultDomain: "infrastructure",
|
|
6529
|
+
summary: outcome.verdict === "codec-split" ? "participants of one call are using different codecs \u2014 an SFU that forwards without transcoding cannot serve all of them" : "the deployment is consistently negotiating a codec other than the expected one",
|
|
6530
|
+
recommendation: outcome.verdict === "codec-split" ? "check codec preferences and any SDP munging; a split usually means one endpoint could not negotiate the preferred codec and the others were not renegotiated with it" : "check codec preferences and endpoint support \u2014 a silent fallback keeps working, at a higher bitrate than you budgeted for",
|
|
6531
|
+
confidence: 0.85
|
|
6532
|
+
},
|
|
6361
6533
|
payload: {
|
|
6362
|
-
type: CODEC_MISMATCH_ISSUE,
|
|
6363
6534
|
verdict: outcome.verdict,
|
|
6364
6535
|
checks: this._checks,
|
|
6365
|
-
evidence: outcome.evidence
|
|
6366
|
-
conclusion: {
|
|
6367
|
-
faultDomain: "infrastructure",
|
|
6368
|
-
summary: outcome.verdict === "codec-split" ? "participants of one call are using different codecs \u2014 an SFU that forwards without transcoding cannot serve all of them" : "the deployment is consistently negotiating a codec other than the expected one",
|
|
6369
|
-
recommendation: outcome.verdict === "codec-split" ? "check codec preferences and any SDP munging; a split usually means one endpoint could not negotiate the preferred codec and the others were not renegotiated with it" : "check codec preferences and endpoint support \u2014 a silent fallback keeps working, at a higher bitrate than you budgeted for",
|
|
6370
|
-
confidence: 0.85
|
|
6371
|
-
}
|
|
6536
|
+
evidence: outcome.evidence
|
|
6372
6537
|
}
|
|
6373
6538
|
});
|
|
6374
6539
|
}
|
|
@@ -6382,7 +6547,7 @@ function mediaKindOf(mimeType) {
|
|
|
6382
6547
|
}
|
|
6383
6548
|
|
|
6384
6549
|
// src/Observer.ts
|
|
6385
|
-
var
|
|
6550
|
+
var logger9 = createLogger("Observer");
|
|
6386
6551
|
var Observer = class extends import_events6.EventEmitter {
|
|
6387
6552
|
observedTURN = new ObservedTURN();
|
|
6388
6553
|
observedCalls = /* @__PURE__ */ new Map();
|
|
@@ -6438,6 +6603,14 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6438
6603
|
* ```
|
|
6439
6604
|
*/
|
|
6440
6605
|
callDetectorConfigs = /* @__PURE__ */ new Map();
|
|
6606
|
+
/**
|
|
6607
|
+
* Owns every call's summary: the resolved `config.callSummary`, the bus subscriptions that keep
|
|
6608
|
+
* the summaries current (one per event type, not one per call), and the summaries themselves.
|
|
6609
|
+
*
|
|
6610
|
+
* `undefined` when `config.callSummary` was absent or `null` — so its presence *is* the answer to
|
|
6611
|
+
* "are summaries on", and nothing is subscribed to anything.
|
|
6612
|
+
*/
|
|
6613
|
+
callSummaryCollector;
|
|
6441
6614
|
constructor(config = {}) {
|
|
6442
6615
|
super();
|
|
6443
6616
|
this.setMaxListeners(Infinity);
|
|
@@ -6449,6 +6622,12 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6449
6622
|
closeClientIfIdleForMs: 6e4,
|
|
6450
6623
|
...config
|
|
6451
6624
|
};
|
|
6625
|
+
if (this.config.callSummary) {
|
|
6626
|
+
this.callSummaryCollector = new CallSummaryCollector(this, {
|
|
6627
|
+
...defaultCallSummaryConfig,
|
|
6628
|
+
...this.config.callSummary
|
|
6629
|
+
});
|
|
6630
|
+
}
|
|
6452
6631
|
}
|
|
6453
6632
|
get numberOfCalls() {
|
|
6454
6633
|
return this.observedCalls.size;
|
|
@@ -6487,7 +6666,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6487
6666
|
break;
|
|
6488
6667
|
}
|
|
6489
6668
|
default: {
|
|
6490
|
-
|
|
6669
|
+
logger9.warn("Unknown detector name %s; skipping", name);
|
|
6491
6670
|
return this;
|
|
6492
6671
|
}
|
|
6493
6672
|
}
|
|
@@ -6572,7 +6751,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6572
6751
|
break;
|
|
6573
6752
|
}
|
|
6574
6753
|
if (!validator) {
|
|
6575
|
-
|
|
6754
|
+
logger9.warn("Unknown validator name %s; skipping", name);
|
|
6576
6755
|
return this;
|
|
6577
6756
|
}
|
|
6578
6757
|
this.validators.add(validator);
|
|
@@ -6603,7 +6782,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6603
6782
|
try {
|
|
6604
6783
|
validator.cancel(reason);
|
|
6605
6784
|
} catch (err) {
|
|
6606
|
-
|
|
6785
|
+
logger9.warn("Error cancelling validator %s: %o", validator.name, err);
|
|
6607
6786
|
}
|
|
6608
6787
|
}
|
|
6609
6788
|
return matching.length;
|
|
@@ -6612,19 +6791,24 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6612
6791
|
if (this.closed || !this.observedCalls.has(callId)) return;
|
|
6613
6792
|
return this.observedCalls.get(callId);
|
|
6614
6793
|
}
|
|
6615
|
-
|
|
6794
|
+
/**
|
|
6795
|
+
* @param acceptCtx the `accept()` context, when this call is being created to receive a sample.
|
|
6796
|
+
* Passed on to `ObserverConfig.createCallAppData`, so the factory can read whatever the caller (or
|
|
6797
|
+
* an accept middleware) put there — a tenant, a region, a trace id.
|
|
6798
|
+
*/
|
|
6799
|
+
createObservedCall(settings, acceptCtx) {
|
|
6616
6800
|
if (this.closed) {
|
|
6617
|
-
|
|
6801
|
+
logger9.warn("Attempted to create a call (callId: %s) on a closed observer", settings.callId);
|
|
6618
6802
|
return void 0;
|
|
6619
6803
|
}
|
|
6620
6804
|
if (this.observedCalls.has(settings.callId)) {
|
|
6621
|
-
|
|
6805
|
+
logger9.warn("Observed Call with id %s already exists; returning the existing instance", settings.callId);
|
|
6622
6806
|
return this.observedCalls.get(settings.callId);
|
|
6623
6807
|
}
|
|
6624
6808
|
const callSettings = {
|
|
6625
6809
|
...settings,
|
|
6626
6810
|
closeCallIfEmptyForMs: settings.closeCallIfEmptyForMs ?? this.config.closeCallIfEmptyForMs,
|
|
6627
|
-
appData: settings.appData ?? this.config.createCallAppData?.({ callId: settings.callId, observer: this })
|
|
6811
|
+
appData: settings.appData ?? this.config.createCallAppData?.({ callId: settings.callId, observer: this, acceptCtx })
|
|
6628
6812
|
};
|
|
6629
6813
|
const callActiveIssuesRegistry = new ActiveIssuesRegistry(this.activeIssuesRegistry);
|
|
6630
6814
|
const observedCall = new ObservedCall(
|
|
@@ -6633,6 +6817,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6633
6817
|
callActiveIssuesRegistry
|
|
6634
6818
|
);
|
|
6635
6819
|
observedCall.remoteTrackResolver = this.config.createRemoteTrackResolver?.(observedCall);
|
|
6820
|
+
observedCall.enableSummary();
|
|
6636
6821
|
for (const [name, detectorConfig] of this.callDetectorConfigs) {
|
|
6637
6822
|
observedCall.addDetector(name, detectorConfig);
|
|
6638
6823
|
}
|
|
@@ -6650,16 +6835,16 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6650
6835
|
this._notify("call-added", { ...this.eventScope, observedCall });
|
|
6651
6836
|
return observedCall;
|
|
6652
6837
|
}
|
|
6653
|
-
getOrCreateObservedCall(settings) {
|
|
6654
|
-
return this.getObservedCall(settings.callId) ?? this.createObservedCall(settings);
|
|
6838
|
+
getOrCreateObservedCall(settings, acceptCtx) {
|
|
6839
|
+
return this.getObservedCall(settings.callId) ?? this.createObservedCall(settings, acceptCtx);
|
|
6655
6840
|
}
|
|
6656
6841
|
createObservedMediasoupRouter(settings) {
|
|
6657
6842
|
if (this.closed) {
|
|
6658
|
-
|
|
6843
|
+
logger9.warn("Attempted to create mediasoup router (id: %d) on a closed observer", settings.router.id);
|
|
6659
6844
|
return void 0;
|
|
6660
6845
|
}
|
|
6661
6846
|
if (this.observedMediasoupRouters.has(settings.router.id)) {
|
|
6662
|
-
|
|
6847
|
+
logger9.warn("Observed Mediasoup Router (id %s) already exists; returning the existing instance", settings.router.id);
|
|
6663
6848
|
return this.observedMediasoupRouters.get(settings.router.id);
|
|
6664
6849
|
}
|
|
6665
6850
|
const observedMediasoupRouter = new ObservedMediasoupRouter(settings);
|
|
@@ -6690,13 +6875,14 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6690
6875
|
}
|
|
6691
6876
|
close() {
|
|
6692
6877
|
if (this.closed) {
|
|
6693
|
-
return
|
|
6878
|
+
return logger9.debug("Attempted to close twice");
|
|
6694
6879
|
}
|
|
6695
6880
|
this.closed = true;
|
|
6696
6881
|
for (const call of [...this.observedCalls.values()]) call.close();
|
|
6697
6882
|
this.detectors.clear();
|
|
6698
6883
|
for (const validator of [...this.validators]) validator.cancel("observer closed");
|
|
6699
6884
|
this.validators.clear();
|
|
6885
|
+
this.callSummaryCollector?.close();
|
|
6700
6886
|
this.activeIssuesRegistry.clear();
|
|
6701
6887
|
this._notify("observer-closed", { ...this.eventScope });
|
|
6702
6888
|
}
|
|
@@ -6707,7 +6893,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6707
6893
|
try {
|
|
6708
6894
|
this.acceptMiddlewares.process({ sample, context });
|
|
6709
6895
|
} catch (err) {
|
|
6710
|
-
|
|
6896
|
+
logger9.warn("An accept middleware threw; dropping the sample. %o", err);
|
|
6711
6897
|
}
|
|
6712
6898
|
if (!sample.callId) {
|
|
6713
6899
|
this._notify("sample-rejected", { ...this.eventScope, reason: "missing-callId", sample });
|
|
@@ -6717,21 +6903,9 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6717
6903
|
this._notify("sample-rejected", { ...this.eventScope, reason: "missing-clientId", sample });
|
|
6718
6904
|
return;
|
|
6719
6905
|
}
|
|
6720
|
-
|
|
6721
|
-
if (!call) {
|
|
6722
|
-
call = this.createObservedCall({
|
|
6723
|
-
callId: sample.callId,
|
|
6724
|
-
appData: this.config.createCallAppData?.({ callId: sample.callId, observer: this })
|
|
6725
|
-
});
|
|
6726
|
-
}
|
|
6906
|
+
const call = this.getOrCreateObservedCall({ callId: sample.callId }, context);
|
|
6727
6907
|
if (!call) return;
|
|
6728
|
-
|
|
6729
|
-
if (!client) {
|
|
6730
|
-
client = call.createObservedClient({
|
|
6731
|
-
clientId: sample.clientId,
|
|
6732
|
-
appData: this.config.createClientAppData?.({ clientId: sample.clientId, observedCall: call })
|
|
6733
|
-
});
|
|
6734
|
-
}
|
|
6908
|
+
const client = call.getOrCreateObservedClient({ clientId: sample.clientId }, context);
|
|
6735
6909
|
if (!client) return;
|
|
6736
6910
|
client.accept(sample, context);
|
|
6737
6911
|
}
|
|
@@ -6759,7 +6933,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6759
6933
|
try {
|
|
6760
6934
|
validator.update();
|
|
6761
6935
|
} catch (err) {
|
|
6762
|
-
|
|
6936
|
+
logger9.warn("Error running validator %s: %o", validator.name, err);
|
|
6763
6937
|
}
|
|
6764
6938
|
}
|
|
6765
6939
|
this._notify("observer-updated", { ...this.eventScope });
|
|
@@ -6772,7 +6946,7 @@ var Observer = class extends import_events6.EventEmitter {
|
|
|
6772
6946
|
*/
|
|
6773
6947
|
addIssue(issue) {
|
|
6774
6948
|
if (this.closed) return;
|
|
6775
|
-
this._notify("observer-issue", { ...this.eventScope, issue });
|
|
6949
|
+
this._notify("observer-issue", { ...this.eventScope, issue: { ...issue, scope: "observer" } });
|
|
6776
6950
|
}
|
|
6777
6951
|
/** Emit an Observer-bus event. */
|
|
6778
6952
|
_notify(type, ...args) {
|
|
@@ -6890,18 +7064,11 @@ var CallHealthAggregator = class {
|
|
|
6890
7064
|
}
|
|
6891
7065
|
};
|
|
6892
7066
|
|
|
6893
|
-
// src/common/
|
|
6894
|
-
function issuePayloadOf(issue) {
|
|
6895
|
-
const payload = issue.payload;
|
|
6896
|
-
if (payload === void 0) return void 0;
|
|
6897
|
-
return typeof payload === "string" ? parseJsonObject(payload) : payload;
|
|
6898
|
-
}
|
|
7067
|
+
// src/common/Issue.ts
|
|
6899
7068
|
function issuePayloadAsString(issue) {
|
|
6900
|
-
|
|
6901
|
-
if (payload === void 0) return void 0;
|
|
6902
|
-
if (typeof payload === "string") return payload;
|
|
7069
|
+
if (issue.payload === void 0) return void 0;
|
|
6903
7070
|
try {
|
|
6904
|
-
return JSON.stringify(payload);
|
|
7071
|
+
return JSON.stringify(issue.payload);
|
|
6905
7072
|
} catch {
|
|
6906
7073
|
return void 0;
|
|
6907
7074
|
}
|
|
@@ -7196,6 +7363,7 @@ function createP2pRemoteTrackResolverFactory() {
|
|
|
7196
7363
|
CallConcurrentIssueDetector,
|
|
7197
7364
|
CallConcurrentIssueTypes,
|
|
7198
7365
|
CallHealthAggregator,
|
|
7366
|
+
CallSummaryCollector,
|
|
7199
7367
|
ClientEventTypes,
|
|
7200
7368
|
ClientMetaTypes,
|
|
7201
7369
|
ClientPopulationIssueDetector,
|
|
@@ -7254,16 +7422,17 @@ function createP2pRemoteTrackResolverFactory() {
|
|
|
7254
7422
|
concludeObserverIssue,
|
|
7255
7423
|
correlation,
|
|
7256
7424
|
counterDelta,
|
|
7425
|
+
createCallSummary,
|
|
7257
7426
|
createDefaultMediasoupRemoteTrackResolverFactory,
|
|
7258
7427
|
createInMemorySink,
|
|
7259
7428
|
createJsonlFileSink,
|
|
7260
7429
|
createJsonlFileSinkFactory,
|
|
7261
7430
|
createLogger,
|
|
7262
7431
|
createP2pRemoteTrackResolverFactory,
|
|
7432
|
+
defaultCallSummaryConfig,
|
|
7263
7433
|
defaultClientHealthThresholds,
|
|
7264
7434
|
isClientIssueResolutionEntry,
|
|
7265
7435
|
issuePayloadAsString,
|
|
7266
|
-
issuePayloadOf,
|
|
7267
7436
|
mannKendall,
|
|
7268
7437
|
mannKendallVerdict,
|
|
7269
7438
|
median,
|