@webex/internal-plugin-metrics 3.12.0-next.34 → 3.12.0-next.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/call-diagnostic/call-diagnostic-metrics-latencies.js +571 -15
- package/dist/call-diagnostic/call-diagnostic-metrics-latencies.js.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/metrics.js +1 -1
- package/dist/metrics.types.js +4 -0
- package/dist/metrics.types.js.map +1 -1
- package/dist/types/call-diagnostic/call-diagnostic-metrics-latencies.d.ts +199 -5
- package/dist/types/index.d.ts +2 -2
- package/dist/types/metrics.types.d.ts +4 -2
- package/package.json +2 -2
- package/src/call-diagnostic/call-diagnostic-metrics-latencies.ts +661 -5
- package/src/index.ts +2 -0
- package/src/metrics.types.ts +15 -2
- package/test/unit/spec/call-diagnostic/call-diagnostic-metrics-latencies.ts +1192 -16
|
@@ -3,10 +3,58 @@
|
|
|
3
3
|
import {WebexPlugin} from '@webex/webex-core';
|
|
4
4
|
import {clamp} from 'lodash';
|
|
5
5
|
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
LOCUS_SYNC_LATENCY_EVENT_NAMES,
|
|
8
|
+
MetricEventNames,
|
|
9
|
+
PreComputedLatencies,
|
|
10
|
+
} from '../metrics.types';
|
|
7
11
|
|
|
8
12
|
// we only care about client event and feature event for now
|
|
9
13
|
|
|
14
|
+
type LocusSyncLatencyMilestone = {
|
|
15
|
+
meetingId: string;
|
|
16
|
+
dataSetName: string;
|
|
17
|
+
key: MetricEventNames;
|
|
18
|
+
value: number;
|
|
19
|
+
trackingId?: string;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
type SaveTimestampOptions = {
|
|
23
|
+
meetingId?: string;
|
|
24
|
+
dataSetName?: string;
|
|
25
|
+
trackingId?: string;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
type SaveLatencyOptions = {
|
|
29
|
+
accumulate?: boolean;
|
|
30
|
+
meetingId?: string;
|
|
31
|
+
dataSetName?: string;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
type LocusSyncLatencyRecord = {
|
|
35
|
+
meetingId: string;
|
|
36
|
+
dataSetName: string;
|
|
37
|
+
randomBackoffTime: number;
|
|
38
|
+
trackingId?: string;
|
|
39
|
+
syncStart?: number;
|
|
40
|
+
hashTreeRequest?: number;
|
|
41
|
+
hashTreeResponse?: number;
|
|
42
|
+
syncRequest?: number;
|
|
43
|
+
syncResponse?: number;
|
|
44
|
+
messageReceived?: number;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
type LocusSyncLatencyTimestampKey = Exclude<keyof LocusSyncLatencyRecord, 'randomBackoffTime'>;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Container for the latencies tracked for a single meeting. Locus sync is currently the only
|
|
51
|
+
* meeting-specific latency we track, but this wrapper lets us add other latency records in the
|
|
52
|
+
* future without changing the shape of `meetingLatencies`.
|
|
53
|
+
*/
|
|
54
|
+
type MeetingLatencyRecord = {
|
|
55
|
+
locusSync: LocusSyncLatencyRecord;
|
|
56
|
+
};
|
|
57
|
+
|
|
10
58
|
/**
|
|
11
59
|
* @description Helper class to store latencies timestamp and to calculate various latencies for CA.
|
|
12
60
|
* @exports
|
|
@@ -15,10 +63,24 @@ import {MetricEventNames, PreComputedLatencies} from '../metrics.types';
|
|
|
15
63
|
export default class CallDiagnosticLatencies extends WebexPlugin {
|
|
16
64
|
latencyTimestamps: Map<MetricEventNames, number>;
|
|
17
65
|
precomputedLatencies: Map<PreComputedLatencies, number>;
|
|
66
|
+
meetingLatencies: Map<string, MeetingLatencyRecord[]>;
|
|
18
67
|
// meetingId that the current latencies are for
|
|
19
68
|
private meetingId?: string;
|
|
20
69
|
private MAX_INTEGER = 2147483647;
|
|
21
70
|
|
|
71
|
+
// Aligned with UCF desktop (MekleFlowSyncMetricsTracker::kSleepWakeSkewThresholdMs): if any
|
|
72
|
+
// measured Locus sync segment exceeds this threshold the record is assumed to be corrupted by a
|
|
73
|
+
// clock jump (sleep/wake) and is discarded instead of being reported.
|
|
74
|
+
private LOCUS_SYNC_SKEW_THRESHOLD_MS = 10 * 60 * 1000;
|
|
75
|
+
|
|
76
|
+
// Incomplete Locus sync records (missing the /sync response and/or the LLM message) are kept for
|
|
77
|
+
// this long so a late milestone can still complete them, then pruned as stale.
|
|
78
|
+
private LOCUS_SYNC_PENDING_RECORD_TTL_MS = 5 * 60 * 1000;
|
|
79
|
+
|
|
80
|
+
// Safety cap on the number of records kept per meeting + dataset, to bound memory if syncs churn
|
|
81
|
+
// faster than the TTL can clean them up.
|
|
82
|
+
private LOCUS_SYNC_MAX_RECORDS_PER_DATASET = 20;
|
|
83
|
+
|
|
22
84
|
/**
|
|
23
85
|
* @constructor
|
|
24
86
|
*/
|
|
@@ -26,6 +88,7 @@ export default class CallDiagnosticLatencies extends WebexPlugin {
|
|
|
26
88
|
super(...args);
|
|
27
89
|
this.latencyTimestamps = new Map();
|
|
28
90
|
this.precomputedLatencies = new Map();
|
|
91
|
+
this.meetingLatencies = new Map();
|
|
29
92
|
}
|
|
30
93
|
|
|
31
94
|
/**
|
|
@@ -34,6 +97,566 @@ export default class CallDiagnosticLatencies extends WebexPlugin {
|
|
|
34
97
|
public clearTimestamps() {
|
|
35
98
|
this.latencyTimestamps.clear();
|
|
36
99
|
this.precomputedLatencies.clear();
|
|
100
|
+
this.meetingLatencies.clear();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Clear tracked Locus sync latency state for a dataset.
|
|
105
|
+
*
|
|
106
|
+
* When a `trackingId` is supplied only the single record matching it is dropped, mirroring the
|
|
107
|
+
* native client which discards the individual in-progress record on a sync error (by syncId).
|
|
108
|
+
* When no `trackingId` is supplied every record for the dataset is dropped (legacy behavior).
|
|
109
|
+
*
|
|
110
|
+
* @param dataSetName dataset name
|
|
111
|
+
* @param meetingId meeting id
|
|
112
|
+
* @param trackingId optional sync tracking id to drop a single record instead of the whole dataset
|
|
113
|
+
*/
|
|
114
|
+
public clearLocusSyncLatency(dataSetName: string, meetingId: string, trackingId?: string) {
|
|
115
|
+
const records = this.meetingLatencies.get(meetingId);
|
|
116
|
+
|
|
117
|
+
if (!records) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const remainingRecords = trackingId
|
|
122
|
+
? records.filter((record) => record.locusSync.trackingId !== trackingId)
|
|
123
|
+
: records.filter((record) => record.locusSync.dataSetName !== dataSetName);
|
|
124
|
+
|
|
125
|
+
this.setMeetingLatencyRecords(meetingId, remainingRecords);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Calculates Locus sync latency values from stored milestone timestamps.
|
|
130
|
+
* @param meetingId meeting id
|
|
131
|
+
* @param trackingId sync tracking id used to match the record
|
|
132
|
+
* @returns sync latency metrics
|
|
133
|
+
*/
|
|
134
|
+
public getLocusSyncLatency(meetingId: string, trackingId: string) {
|
|
135
|
+
if (!trackingId) {
|
|
136
|
+
return undefined;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const record = this.getLocusSyncLatencyRecord(meetingId, trackingId);
|
|
140
|
+
|
|
141
|
+
if (!record) {
|
|
142
|
+
return undefined;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Some sync flows skip the /hashtree request, for example single-leaf data sets or cases
|
|
146
|
+
// where we already know which leaves to sync. Treat the missing hashtree segment as 0,
|
|
147
|
+
// while still requiring the sync request/response/message milestones below.
|
|
148
|
+
const hashtreePrepTime =
|
|
149
|
+
this.getDiffBetweenLocusSyncTimestamps(record, 'syncStart', 'hashTreeRequest') ?? 0;
|
|
150
|
+
const hashtreeResponseTime =
|
|
151
|
+
this.getDiffBetweenLocusSyncTimestamps(record, 'hashTreeRequest', 'hashTreeResponse') ?? 0;
|
|
152
|
+
const syncPrepStart = this.getLocusSyncPrepStart(record);
|
|
153
|
+
const syncPrepTime = this.getDiffBetweenLocusSyncTimestamps(
|
|
154
|
+
record,
|
|
155
|
+
syncPrepStart,
|
|
156
|
+
'syncRequest'
|
|
157
|
+
);
|
|
158
|
+
const syncResponseTime = this.getDiffBetweenLocusSyncTimestamps(
|
|
159
|
+
record,
|
|
160
|
+
'syncRequest',
|
|
161
|
+
'syncResponse'
|
|
162
|
+
);
|
|
163
|
+
// Aligned with UCF: syncMessageReceiveTime is measured from the sync request start (not the
|
|
164
|
+
// sync response), i.e. how long after we asked Locus to sync the resulting state update arrived.
|
|
165
|
+
const syncMessageReceiveTime = this.getDiffBetweenLocusSyncTimestamps(
|
|
166
|
+
record,
|
|
167
|
+
'syncRequest',
|
|
168
|
+
'messageReceived'
|
|
169
|
+
);
|
|
170
|
+
const totalTimeFromMessageReceived = this.getDiffBetweenLocusSyncTimestamps(
|
|
171
|
+
record,
|
|
172
|
+
'syncStart',
|
|
173
|
+
'messageReceived'
|
|
174
|
+
);
|
|
175
|
+
const totalTimeFromSyncResponse = this.getDiffBetweenLocusSyncTimestamps(
|
|
176
|
+
record,
|
|
177
|
+
'syncStart',
|
|
178
|
+
'syncResponse'
|
|
179
|
+
);
|
|
180
|
+
|
|
181
|
+
// messageReceived can occasionally be out of order due to race conditions relative to the
|
|
182
|
+
// /sync response. Use the larger of the two so totalTime is never less than syncStart->syncResponse.
|
|
183
|
+
const totalTime =
|
|
184
|
+
typeof totalTimeFromMessageReceived === 'number' &&
|
|
185
|
+
typeof totalTimeFromSyncResponse === 'number'
|
|
186
|
+
? Math.max(totalTimeFromMessageReceived, totalTimeFromSyncResponse)
|
|
187
|
+
: totalTimeFromMessageReceived ?? totalTimeFromSyncResponse;
|
|
188
|
+
|
|
189
|
+
if (typeof syncPrepTime !== 'number' || typeof totalTime !== 'number') {
|
|
190
|
+
return undefined;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Aligned with UCF desktop (MekleFlowSyncMetricsTracker kSleepWakeSkewThresholdMs): if any
|
|
194
|
+
// individual segment is larger than the skew threshold the timestamps were almost certainly
|
|
195
|
+
// distorted by a clock jump (device sleep/wake), so discard the whole record.
|
|
196
|
+
const skewedSegment = [
|
|
197
|
+
record.randomBackoffTime,
|
|
198
|
+
hashtreePrepTime,
|
|
199
|
+
hashtreeResponseTime,
|
|
200
|
+
syncPrepTime,
|
|
201
|
+
syncResponseTime,
|
|
202
|
+
syncMessageReceiveTime,
|
|
203
|
+
totalTime,
|
|
204
|
+
].some((segment) => typeof segment === 'number' && segment > this.LOCUS_SYNC_SKEW_THRESHOLD_MS);
|
|
205
|
+
|
|
206
|
+
if (skewedSegment) {
|
|
207
|
+
return undefined;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// Emit whenever at least one measured segment is non-zero. The LLM broadcast itself is gated
|
|
211
|
+
// in completeLocusSyncLatency(); this only guards against an all-zero (unmeasurable) record.
|
|
212
|
+
const hasMeaningfulSegment =
|
|
213
|
+
hashtreeResponseTime > 0 ||
|
|
214
|
+
(typeof syncResponseTime === 'number' && syncResponseTime > 0) ||
|
|
215
|
+
(typeof syncMessageReceiveTime === 'number' && syncMessageReceiveTime > 0);
|
|
216
|
+
|
|
217
|
+
if (!hasMeaningfulSegment) {
|
|
218
|
+
return undefined;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
return {
|
|
222
|
+
randomBackoffTime: this.getClampedLocusSyncLatency(record.randomBackoffTime),
|
|
223
|
+
hashtreePrepTime,
|
|
224
|
+
hashtreeResponseTime,
|
|
225
|
+
syncPrepTime,
|
|
226
|
+
...(typeof syncResponseTime === 'number' ? {syncResponseTime} : {}),
|
|
227
|
+
...(typeof syncMessageReceiveTime === 'number' ? {syncMessageReceiveTime} : {}),
|
|
228
|
+
totalTime,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Records the time the LLM state-update message for a Locus sync arrived. This is the milestone
|
|
234
|
+
* that gates the client.locus.sync.complete metric: the metric is emitted only for flows where
|
|
235
|
+
* the update came over LLM (aligned with the agreed requirement that body/http-only syncs do not
|
|
236
|
+
* emit). The record is matched by meeting id + tracking id, regardless of which other milestones
|
|
237
|
+
* are present yet, so the real arrival time is captured even if the /sync response has not landed.
|
|
238
|
+
* @param meetingId meeting id
|
|
239
|
+
* @param trackingId sync tracking id used to match the pending record
|
|
240
|
+
* @returns void
|
|
241
|
+
*/
|
|
242
|
+
public recordLocusSyncMessageReceived(meetingId: string, trackingId: string) {
|
|
243
|
+
if (!trackingId) {
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
const record = this.getLocusSyncLatencyRecord(meetingId, trackingId);
|
|
248
|
+
|
|
249
|
+
if (record && typeof record.messageReceived !== 'number') {
|
|
250
|
+
record.messageReceived = new Date().getTime();
|
|
251
|
+
const records = this.meetingLatencies.get(meetingId);
|
|
252
|
+
|
|
253
|
+
if (records) {
|
|
254
|
+
this.setMeetingLatencyRecords(meetingId, records, {
|
|
255
|
+
changedRecord: record,
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Complete and remove the Locus sync latency record for a meeting + tracking id.
|
|
263
|
+
*
|
|
264
|
+
* Both milestones are required, in any arrival order: the metric is emitted only once BOTH the
|
|
265
|
+
* matching LLM broadcast (messageReceived) AND the /sync response have arrived. The LLM broadcast
|
|
266
|
+
* confirms the client that issued the /sync received the resulting state update for its tracking
|
|
267
|
+
* id (so sync cancel, or a sync storm where the final broadcast echoed another client's tracking
|
|
268
|
+
* id, never emit); the /sync response is always waited for too, since the LLM message can arrive
|
|
269
|
+
* before the HTTP response. If either milestone is missing the record is kept so the other can
|
|
270
|
+
* still arrive; it is only consumed when the metric is actually emitted.
|
|
271
|
+
* @param meetingId meeting id
|
|
272
|
+
* @param trackingId sync tracking id used to match the pending record
|
|
273
|
+
* @returns completed sync latency metric payload, or undefined when not completed
|
|
274
|
+
*/
|
|
275
|
+
public completeLocusSyncLatency(meetingId: string, trackingId: string) {
|
|
276
|
+
if (!trackingId) {
|
|
277
|
+
return undefined;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
const record = this.getLocusSyncLatencyRecord(meetingId, trackingId);
|
|
281
|
+
|
|
282
|
+
if (!record) {
|
|
283
|
+
return undefined;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
const hasMessageReceived = typeof record.messageReceived === 'number';
|
|
287
|
+
const hasSyncResponse = typeof record.syncResponse === 'number';
|
|
288
|
+
|
|
289
|
+
// Wait until both milestones are present before emitting. The LLM broadcast and the /sync
|
|
290
|
+
// response can arrive in either order, so we keep the record until the second one lands.
|
|
291
|
+
if (!hasMessageReceived || !hasSyncResponse) {
|
|
292
|
+
return undefined;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
const syncLatency = this.getLocusSyncLatency(meetingId, trackingId);
|
|
296
|
+
|
|
297
|
+
this.removeLocusSyncLatencyRecord(meetingId, trackingId);
|
|
298
|
+
|
|
299
|
+
if (!syncLatency) {
|
|
300
|
+
return undefined;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
const completed = {
|
|
304
|
+
dataSet: record.dataSetName,
|
|
305
|
+
syncLatency,
|
|
306
|
+
};
|
|
307
|
+
|
|
308
|
+
return completed;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Helper to calculate end - start for Locus sync milestones.
|
|
313
|
+
* @param record tracked milestone timestamps
|
|
314
|
+
* @param a start milestone
|
|
315
|
+
* @param b end milestone
|
|
316
|
+
* @returns latency
|
|
317
|
+
*/
|
|
318
|
+
private getDiffBetweenLocusSyncTimestamps(
|
|
319
|
+
record: LocusSyncLatencyRecord,
|
|
320
|
+
a: LocusSyncLatencyTimestampKey,
|
|
321
|
+
b: LocusSyncLatencyTimestampKey
|
|
322
|
+
) {
|
|
323
|
+
const start = record[a];
|
|
324
|
+
const end = record[b];
|
|
325
|
+
|
|
326
|
+
if (typeof start !== 'number' || typeof end !== 'number') {
|
|
327
|
+
return undefined;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
return this.getClampedLocusSyncLatency(end - start);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* Get the timestamp that starts the sync prep segment.
|
|
335
|
+
* @param record tracked milestone timestamps
|
|
336
|
+
* @returns sync prep start milestone
|
|
337
|
+
*/
|
|
338
|
+
private getLocusSyncPrepStart(record: LocusSyncLatencyRecord): LocusSyncLatencyTimestampKey {
|
|
339
|
+
return typeof record.hashTreeResponse === 'number' ? 'hashTreeResponse' : 'syncStart';
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Round and clamp Locus sync latency values.
|
|
344
|
+
* @param latency latency value
|
|
345
|
+
* @returns rounded latency
|
|
346
|
+
*/
|
|
347
|
+
private getClampedLocusSyncLatency(latency: number) {
|
|
348
|
+
return Math.round(clamp(latency, 0, this.MAX_INTEGER));
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* Finds a Locus sync latency record for a meeting using a predicate run against each stored
|
|
353
|
+
* locusSync record.
|
|
354
|
+
* @param meetingId meeting id
|
|
355
|
+
* @param predicate matcher run against each locusSync record
|
|
356
|
+
* @param options.searchFromLatest when true, iterate from the most recently added record first
|
|
357
|
+
* @returns matching Locus sync latency record
|
|
358
|
+
*/
|
|
359
|
+
private findLocusSyncRecord(
|
|
360
|
+
meetingId: string,
|
|
361
|
+
predicate: (locusSync: LocusSyncLatencyRecord) => boolean,
|
|
362
|
+
{searchFromLatest = false}: {searchFromLatest?: boolean} = {}
|
|
363
|
+
) {
|
|
364
|
+
const records = this.meetingLatencies.get(meetingId);
|
|
365
|
+
|
|
366
|
+
if (!records) {
|
|
367
|
+
return undefined;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
const orderedRecords = searchFromLatest ? [...records].reverse() : records;
|
|
371
|
+
|
|
372
|
+
return orderedRecords.find((record) => predicate(record.locusSync))?.locusSync;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* Get the Locus sync latency record for a meeting and tracking id.
|
|
377
|
+
* @param meetingId meeting id
|
|
378
|
+
* @param trackingId /sync response tracking id
|
|
379
|
+
* @returns matching Locus sync latency record
|
|
380
|
+
*/
|
|
381
|
+
private getLocusSyncLatencyRecord(meetingId: string, trackingId: string) {
|
|
382
|
+
return this.findLocusSyncRecord(meetingId, (locusSync) => locusSync.trackingId === trackingId);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Remove the Locus sync latency record for a meeting and tracking id.
|
|
387
|
+
* @param meetingId meeting id
|
|
388
|
+
* @param trackingId sync tracking id used to match the record
|
|
389
|
+
* @returns void
|
|
390
|
+
*/
|
|
391
|
+
private removeLocusSyncLatencyRecord(meetingId: string, trackingId: string) {
|
|
392
|
+
const records = this.meetingLatencies.get(meetingId);
|
|
393
|
+
|
|
394
|
+
if (!records) {
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
const remainingRecords = records.filter(({locusSync}) => locusSync.trackingId !== trackingId);
|
|
399
|
+
|
|
400
|
+
this.setMeetingLatencyRecords(meetingId, remainingRecords);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* Prune stale never-completed Locus sync latency records for the same dataset as the record that
|
|
405
|
+
* just changed. Records are dropped when they have been pending longer than the TTL, or to keep
|
|
406
|
+
* the number of records per dataset under the safety cap (oldest pending candidates first).
|
|
407
|
+
* @param records current records for the meeting
|
|
408
|
+
* @param changedRecord the record that was just updated, used as the reference point
|
|
409
|
+
* @returns the records to keep
|
|
410
|
+
*/
|
|
411
|
+
private pruneStaleLocusSyncLatencyRecords(
|
|
412
|
+
records: MeetingLatencyRecord[],
|
|
413
|
+
changedRecord: LocusSyncLatencyRecord
|
|
414
|
+
) {
|
|
415
|
+
const currentTimestamp = this.getLatestLocusSyncTimestamp(changedRecord);
|
|
416
|
+
|
|
417
|
+
if (typeof currentTimestamp !== 'number') {
|
|
418
|
+
return records;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// A record is only a cleanup candidate when it belongs to the same dataset as the record that
|
|
422
|
+
// just changed, is not that record itself, and has never completed. Everything else is kept.
|
|
423
|
+
const isPrunableCandidate = (locusSync: LocusSyncLatencyRecord) => {
|
|
424
|
+
if (locusSync.dataSetName !== changedRecord.dataSetName) {
|
|
425
|
+
return false;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
if (changedRecord.trackingId && locusSync.trackingId === changedRecord.trackingId) {
|
|
429
|
+
return false;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
const hasCompleted =
|
|
433
|
+
typeof locusSync.syncResponse === 'number' && typeof locusSync.messageReceived === 'number';
|
|
434
|
+
|
|
435
|
+
return !hasCompleted;
|
|
436
|
+
};
|
|
437
|
+
|
|
438
|
+
// 1) Drop candidates that have been pending longer than the TTL.
|
|
439
|
+
const recordsAfterTtl = records.filter(({locusSync}) => {
|
|
440
|
+
if (!isPrunableCandidate(locusSync)) {
|
|
441
|
+
return true;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
const lastTimestamp = this.getLatestLocusSyncTimestamp(locusSync);
|
|
445
|
+
|
|
446
|
+
if (typeof lastTimestamp !== 'number') {
|
|
447
|
+
return true;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
return currentTimestamp - lastTimestamp < this.LOCUS_SYNC_PENDING_RECORD_TTL_MS;
|
|
451
|
+
});
|
|
452
|
+
|
|
453
|
+
// 2) Cap the number of records kept per dataset, dropping the oldest pending candidates first.
|
|
454
|
+
let overflowCount = Math.max(
|
|
455
|
+
0,
|
|
456
|
+
recordsAfterTtl.filter(({locusSync}) => locusSync.dataSetName === changedRecord.dataSetName)
|
|
457
|
+
.length - this.LOCUS_SYNC_MAX_RECORDS_PER_DATASET
|
|
458
|
+
);
|
|
459
|
+
|
|
460
|
+
if (overflowCount === 0) {
|
|
461
|
+
return recordsAfterTtl;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
return recordsAfterTtl.filter(({locusSync}) => {
|
|
465
|
+
if (overflowCount <= 0 || !isPrunableCandidate(locusSync)) {
|
|
466
|
+
return true;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
overflowCount -= 1;
|
|
470
|
+
|
|
471
|
+
return false;
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Get the most recent milestone timestamp stored on a Locus sync latency record, checking
|
|
477
|
+
* milestones from latest to earliest so the newest available one is returned.
|
|
478
|
+
* @param record tracked milestone timestamps
|
|
479
|
+
* @returns latest milestone timestamp, or undefined when none are set
|
|
480
|
+
*/
|
|
481
|
+
private getLatestLocusSyncTimestamp(record: LocusSyncLatencyRecord) {
|
|
482
|
+
return (
|
|
483
|
+
record.messageReceived ??
|
|
484
|
+
record.syncResponse ??
|
|
485
|
+
record.syncRequest ??
|
|
486
|
+
record.hashTreeResponse ??
|
|
487
|
+
record.hashTreeRequest ??
|
|
488
|
+
record.syncStart
|
|
489
|
+
);
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* Persist the Locus sync latency records for a meeting, deleting the meeting entry when no
|
|
494
|
+
* records remain. When pruneOptions is provided, stale records are pruned before saving.
|
|
495
|
+
* @param meetingId meeting id
|
|
496
|
+
* @param records records to save
|
|
497
|
+
* @param pruneOptions when provided, triggers stale record pruning relative to changedRecord
|
|
498
|
+
* @returns void
|
|
499
|
+
*/
|
|
500
|
+
private setMeetingLatencyRecords(
|
|
501
|
+
meetingId: string,
|
|
502
|
+
records: MeetingLatencyRecord[],
|
|
503
|
+
pruneOptions?: {
|
|
504
|
+
changedRecord: LocusSyncLatencyRecord;
|
|
505
|
+
}
|
|
506
|
+
) {
|
|
507
|
+
const recordsToSave = pruneOptions
|
|
508
|
+
? this.pruneStaleLocusSyncLatencyRecords(records, pruneOptions.changedRecord)
|
|
509
|
+
: records;
|
|
510
|
+
|
|
511
|
+
if (recordsToSave.length > 0) {
|
|
512
|
+
this.meetingLatencies.set(meetingId, recordsToSave);
|
|
513
|
+
} else {
|
|
514
|
+
this.meetingLatencies.delete(meetingId);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* Get the latest Locus sync latency record waiting for sync.start.
|
|
520
|
+
* @param meetingId meeting id
|
|
521
|
+
* @param dataSetName dataset name
|
|
522
|
+
* @returns latest pending Locus sync latency record
|
|
523
|
+
*/
|
|
524
|
+
private getLatestPendingLocusSyncLatencyRecord(meetingId: string, dataSetName: string) {
|
|
525
|
+
return this.findLocusSyncRecord(
|
|
526
|
+
meetingId,
|
|
527
|
+
(locusSync) => locusSync.dataSetName === dataSetName && locusSync.syncStart === undefined,
|
|
528
|
+
{searchFromLatest: true}
|
|
529
|
+
);
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/**
|
|
533
|
+
* Store random backoff latency for the current or next Locus sync latency record.
|
|
534
|
+
* @param meetingId meeting id
|
|
535
|
+
* @param dataSetName dataset name
|
|
536
|
+
* @param randomBackoffTime random backoff latency value
|
|
537
|
+
* @returns void
|
|
538
|
+
*/
|
|
539
|
+
private saveLocusSyncBackoffLatency({
|
|
540
|
+
meetingId,
|
|
541
|
+
dataSetName,
|
|
542
|
+
randomBackoffTime,
|
|
543
|
+
}: {
|
|
544
|
+
meetingId: string;
|
|
545
|
+
dataSetName: string;
|
|
546
|
+
randomBackoffTime: number;
|
|
547
|
+
}) {
|
|
548
|
+
const pendingRecord = this.getLatestPendingLocusSyncLatencyRecord(meetingId, dataSetName);
|
|
549
|
+
|
|
550
|
+
if (pendingRecord) {
|
|
551
|
+
pendingRecord.randomBackoffTime = randomBackoffTime;
|
|
552
|
+
|
|
553
|
+
return;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
const records = this.meetingLatencies.get(meetingId) ?? [];
|
|
557
|
+
|
|
558
|
+
records.push({
|
|
559
|
+
locusSync: {
|
|
560
|
+
meetingId,
|
|
561
|
+
dataSetName,
|
|
562
|
+
randomBackoffTime,
|
|
563
|
+
},
|
|
564
|
+
});
|
|
565
|
+
this.meetingLatencies.set(meetingId, records);
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/**
|
|
569
|
+
* Checks if metric event name is a Locus sync latency milestone.
|
|
570
|
+
* @param key event name
|
|
571
|
+
* @returns whether event is Locus sync latency milestone
|
|
572
|
+
*/
|
|
573
|
+
private isLocusSyncLatencyEvent(key: MetricEventNames) {
|
|
574
|
+
return LOCUS_SYNC_LATENCY_EVENT_NAMES.includes(key as any);
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
/**
|
|
578
|
+
* Stores a Locus sync latency milestone timestamp.
|
|
579
|
+
* @param options options
|
|
580
|
+
*/
|
|
581
|
+
private saveLocusSyncLatencyTimestamp({
|
|
582
|
+
meetingId,
|
|
583
|
+
dataSetName,
|
|
584
|
+
key,
|
|
585
|
+
value,
|
|
586
|
+
trackingId,
|
|
587
|
+
}: LocusSyncLatencyMilestone) {
|
|
588
|
+
if (!trackingId) {
|
|
589
|
+
// @ts-ignore
|
|
590
|
+
this.webex.logger.warn(
|
|
591
|
+
`CallDiagnosticLatencies: saveLocusSyncLatencyTimestamp called without a trackingId for key "${key}"; skipping Locus sync milestone`
|
|
592
|
+
);
|
|
593
|
+
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
if (key === 'internal.client.locus.sync.start') {
|
|
598
|
+
const pendingRecord = this.getLatestPendingLocusSyncLatencyRecord(meetingId, dataSetName);
|
|
599
|
+
|
|
600
|
+
if (pendingRecord) {
|
|
601
|
+
pendingRecord.trackingId = trackingId;
|
|
602
|
+
pendingRecord.syncStart = value;
|
|
603
|
+
this.setMeetingLatencyRecords(meetingId, this.meetingLatencies.get(meetingId) ?? [], {
|
|
604
|
+
changedRecord: pendingRecord,
|
|
605
|
+
});
|
|
606
|
+
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
const records = this.meetingLatencies.get(meetingId) ?? [];
|
|
611
|
+
|
|
612
|
+
records.push({
|
|
613
|
+
locusSync: {
|
|
614
|
+
meetingId,
|
|
615
|
+
dataSetName,
|
|
616
|
+
randomBackoffTime: 0,
|
|
617
|
+
trackingId,
|
|
618
|
+
syncStart: value,
|
|
619
|
+
},
|
|
620
|
+
});
|
|
621
|
+
this.setMeetingLatencyRecords(meetingId, records, {
|
|
622
|
+
changedRecord: records[records.length - 1].locusSync,
|
|
623
|
+
});
|
|
624
|
+
|
|
625
|
+
return;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
// Every sync milestone after sync.start is stamped with the same tracking id that was
|
|
629
|
+
// generated up-front and forced onto the /sync request, so the record is located by
|
|
630
|
+
// meeting id + tracking id.
|
|
631
|
+
const record = this.getLocusSyncLatencyRecord(meetingId, trackingId);
|
|
632
|
+
|
|
633
|
+
if (!record) {
|
|
634
|
+
return;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
switch (key) {
|
|
638
|
+
case 'internal.client.locus.hashtree.request':
|
|
639
|
+
record.hashTreeRequest = value;
|
|
640
|
+
break;
|
|
641
|
+
case 'internal.client.locus.hashtree.response':
|
|
642
|
+
record.hashTreeResponse = value;
|
|
643
|
+
break;
|
|
644
|
+
case 'internal.client.locus.sync.request':
|
|
645
|
+
record.syncRequest = value;
|
|
646
|
+
break;
|
|
647
|
+
case 'internal.client.locus.sync.response':
|
|
648
|
+
record.syncResponse = value;
|
|
649
|
+
break;
|
|
650
|
+
case 'internal.client.locus.sync.message.received':
|
|
651
|
+
record.messageReceived = value;
|
|
652
|
+
break;
|
|
653
|
+
default:
|
|
654
|
+
break;
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
this.setMeetingLatencyRecords(meetingId, this.meetingLatencies.get(meetingId) ?? [], {
|
|
658
|
+
changedRecord: record,
|
|
659
|
+
});
|
|
37
660
|
}
|
|
38
661
|
|
|
39
662
|
/**
|
|
@@ -72,13 +695,26 @@ export default class CallDiagnosticLatencies extends WebexPlugin {
|
|
|
72
695
|
}: {
|
|
73
696
|
key: MetricEventNames;
|
|
74
697
|
value?: number;
|
|
75
|
-
options?:
|
|
698
|
+
options?: SaveTimestampOptions;
|
|
76
699
|
}) {
|
|
77
700
|
// save the meetingId so we can use the meeting object in latency calculations if needed
|
|
78
701
|
const {meetingId} = options;
|
|
79
702
|
if (meetingId) {
|
|
80
703
|
this.setMeetingId(meetingId);
|
|
81
704
|
}
|
|
705
|
+
|
|
706
|
+
if (this.isLocusSyncLatencyEvent(key) && options.dataSetName && options.meetingId) {
|
|
707
|
+
this.saveLocusSyncLatencyTimestamp({
|
|
708
|
+
meetingId: options.meetingId,
|
|
709
|
+
dataSetName: options.dataSetName,
|
|
710
|
+
key,
|
|
711
|
+
value,
|
|
712
|
+
trackingId: options.trackingId,
|
|
713
|
+
});
|
|
714
|
+
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
|
|
82
718
|
// for some events we're only interested in the first timestamp not last
|
|
83
719
|
// as these events can happen multiple times
|
|
84
720
|
if (
|
|
@@ -102,11 +738,31 @@ export default class CallDiagnosticLatencies extends WebexPlugin {
|
|
|
102
738
|
* Store precomputed latency value
|
|
103
739
|
* @param key - key
|
|
104
740
|
* @param value - value
|
|
105
|
-
* @param
|
|
741
|
+
* @param options - store options (a legacy boolean `accumulate` flag is also accepted)
|
|
742
|
+
* @param options.accumulate - when it is true, it overwrites existing value with sum of the current value and the new measurement otherwise just store the new measurement
|
|
743
|
+
* @param options.meetingId - meeting id, only used for Locus sync latency records
|
|
744
|
+
* @param options.dataSetName - dataset name, only used for Locus sync latency records
|
|
106
745
|
* @throws
|
|
107
746
|
* @returns
|
|
108
747
|
*/
|
|
109
|
-
public saveLatency(key: PreComputedLatencies, value: number,
|
|
748
|
+
public saveLatency(key: PreComputedLatencies, value: number, options: SaveLatencyOptions = {}) {
|
|
749
|
+
// Older (untyped) callers may still pass a boolean `accumulate` as the third argument
|
|
750
|
+
// (saveLatency(key, value, true)). Normalize it to the options object before destructuring,
|
|
751
|
+
// otherwise accumulation is silently lost and legacy SDK/plugin consumers underreport latencies.
|
|
752
|
+
const normalizedOptions: SaveLatencyOptions =
|
|
753
|
+
typeof options === 'boolean' ? {accumulate: options} : options;
|
|
754
|
+
const {accumulate = false, meetingId, dataSetName} = normalizedOptions;
|
|
755
|
+
|
|
756
|
+
if (key === 'internal.client.locus.sync.random.backoff' && meetingId && dataSetName) {
|
|
757
|
+
this.saveLocusSyncBackoffLatency({
|
|
758
|
+
meetingId,
|
|
759
|
+
dataSetName,
|
|
760
|
+
randomBackoffTime: value,
|
|
761
|
+
});
|
|
762
|
+
|
|
763
|
+
return;
|
|
764
|
+
}
|
|
765
|
+
|
|
110
766
|
const existingValue = accumulate ? this.precomputedLatencies.get(key) || 0 : 0;
|
|
111
767
|
this.precomputedLatencies.set(key, value + existingValue);
|
|
112
768
|
}
|
|
@@ -126,7 +782,7 @@ export default class CallDiagnosticLatencies extends WebexPlugin {
|
|
|
126
782
|
const start = performance.now();
|
|
127
783
|
|
|
128
784
|
return callback().finally(() => {
|
|
129
|
-
this.saveLatency(key, performance.now() - start, accumulate);
|
|
785
|
+
this.saveLatency(key, performance.now() - start, {accumulate});
|
|
130
786
|
});
|
|
131
787
|
}
|
|
132
788
|
|