@oneuptime/common 11.3.9 → 11.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/AnalyticsModels/MutableMetric.ts +333 -0
- package/Server/Services/AlertService.ts +167 -206
- package/Server/Services/AlertStateTimelineService.ts +121 -1
- package/Server/Services/AnalyticsDatabaseService.ts +19 -23
- package/Server/Services/IncidentService.ts +179 -252
- package/Server/Services/IncidentStateTimelineService.ts +120 -1
- package/Server/Services/Index.ts +2 -0
- package/Server/Services/MetricAggregationService.ts +73 -3
- package/Server/Services/MetricService.ts +228 -0
- package/Server/Services/MutableMetricService.ts +275 -0
- package/Server/Services/TelemetryAttributeService.ts +183 -1
- package/Server/Utils/AnalyticsDatabase/ClusterConfig.ts +2 -0
- package/Server/Utils/AnalyticsDatabase/Statement.ts +2 -0
- package/Tests/Server/Services/AnalyticsDatabaseService.test.ts +145 -0
- package/Tests/Types/Monitor/MonitorStepEntityScope.test.ts +0 -56
- package/Types/AnalyticsDatabase/AnalyticsTableEngine.ts +1 -0
- package/Types/AnalyticsDatabase/AnalyticsTableName.ts +1 -0
- package/Types/Dashboard/DashboardTemplates.ts +267 -7
- package/Types/Monitor/MonitorStepMetricMonitor.ts +0 -27
- package/build/dist/Models/AnalyticsModels/MutableMetric.js +275 -0
- package/build/dist/Models/AnalyticsModels/MutableMetric.js.map +1 -0
- package/build/dist/Server/Services/AlertService.js +133 -170
- package/build/dist/Server/Services/AlertService.js.map +1 -1
- package/build/dist/Server/Services/AlertStateTimelineService.js +97 -0
- package/build/dist/Server/Services/AlertStateTimelineService.js.map +1 -1
- package/build/dist/Server/Services/AnalyticsDatabaseService.js +16 -15
- package/build/dist/Server/Services/AnalyticsDatabaseService.js.map +1 -1
- package/build/dist/Server/Services/IncidentService.js +142 -207
- package/build/dist/Server/Services/IncidentService.js.map +1 -1
- package/build/dist/Server/Services/IncidentStateTimelineService.js +97 -0
- package/build/dist/Server/Services/IncidentStateTimelineService.js.map +1 -1
- package/build/dist/Server/Services/Index.js +2 -0
- package/build/dist/Server/Services/Index.js.map +1 -1
- package/build/dist/Server/Services/MetricAggregationService.js +49 -3
- package/build/dist/Server/Services/MetricAggregationService.js.map +1 -1
- package/build/dist/Server/Services/MetricService.js +155 -0
- package/build/dist/Server/Services/MetricService.js.map +1 -1
- package/build/dist/Server/Services/MutableMetricService.js +182 -0
- package/build/dist/Server/Services/MutableMetricService.js.map +1 -0
- package/build/dist/Server/Services/TelemetryAttributeService.js +131 -6
- package/build/dist/Server/Services/TelemetryAttributeService.js.map +1 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/ClusterConfig.js +2 -0
- package/build/dist/Server/Utils/AnalyticsDatabase/ClusterConfig.js.map +1 -1
- package/build/dist/Server/Utils/AnalyticsDatabase/Statement.js +2 -0
- package/build/dist/Server/Utils/AnalyticsDatabase/Statement.js.map +1 -1
- package/build/dist/Types/AnalyticsDatabase/AnalyticsTableEngine.js +1 -0
- package/build/dist/Types/AnalyticsDatabase/AnalyticsTableEngine.js.map +1 -1
- package/build/dist/Types/AnalyticsDatabase/AnalyticsTableName.js +1 -0
- package/build/dist/Types/AnalyticsDatabase/AnalyticsTableName.js.map +1 -1
- package/build/dist/Types/Dashboard/DashboardTemplates.js +252 -6
- package/build/dist/Types/Dashboard/DashboardTemplates.js.map +1 -1
- package/build/dist/Types/Monitor/MonitorStepMetricMonitor.js +0 -13
- package/build/dist/Types/Monitor/MonitorStepMetricMonitor.js.map +1 -1
- package/package.json +1 -1
|
@@ -35,11 +35,12 @@ import { IsBillingEnabled } from "../EnvironmentConfig";
|
|
|
35
35
|
import logger, { LogAttributes } from "../Utils/Logger";
|
|
36
36
|
import Semaphore, { SemaphoreMutex } from "../Infrastructure/Semaphore";
|
|
37
37
|
import TelemetryUtil from "../Utils/Telemetry/Telemetry";
|
|
38
|
-
import
|
|
38
|
+
import MutableMetricService from "./MutableMetricService";
|
|
39
39
|
import GlobalConfigService from "./GlobalConfigService";
|
|
40
40
|
import GlobalConfig from "../../Models/DatabaseModels/GlobalConfig";
|
|
41
41
|
import OneUptimeDate from "../../Types/Date";
|
|
42
|
-
import
|
|
42
|
+
import MutableMetric from "../../Models/AnalyticsModels/MutableMetric";
|
|
43
|
+
import { MetricPointType } from "../../Models/AnalyticsModels/Metric";
|
|
43
44
|
import ServiceType from "../../Types/Telemetry/ServiceType";
|
|
44
45
|
import AlertMetricType from "../../Types/Alerts/AlertMetricType";
|
|
45
46
|
import AlertFeedService from "./AlertFeedService";
|
|
@@ -1160,6 +1161,7 @@ ${alertSeverity.name}
|
|
|
1160
1161
|
limit: LIMIT_MAX,
|
|
1161
1162
|
skip: 0,
|
|
1162
1163
|
select: {
|
|
1164
|
+
_id: true,
|
|
1163
1165
|
projectId: true,
|
|
1164
1166
|
monitor: {
|
|
1165
1167
|
_id: true,
|
|
@@ -1178,6 +1180,34 @@ ${alertSeverity.name}
|
|
|
1178
1180
|
};
|
|
1179
1181
|
}
|
|
1180
1182
|
|
|
1183
|
+
@CaptureSpan()
|
|
1184
|
+
protected override async onDeleteSuccess(
|
|
1185
|
+
onDelete: OnDelete<Model>,
|
|
1186
|
+
_itemIdsBeforeDelete: ObjectID[],
|
|
1187
|
+
): Promise<OnDelete<Model>> {
|
|
1188
|
+
if (onDelete.carryForward && onDelete.carryForward.alerts) {
|
|
1189
|
+
for (const alert of onDelete.carryForward.alerts) {
|
|
1190
|
+
if (alert.projectId && alert.id) {
|
|
1191
|
+
const metricRetentionDays: number =
|
|
1192
|
+
await this.getMetricRetentionDays();
|
|
1193
|
+
|
|
1194
|
+
await MutableMetricService.tombstoneEntityMetrics({
|
|
1195
|
+
projectId: alert.projectId,
|
|
1196
|
+
primaryEntityId: alert.id,
|
|
1197
|
+
primaryEntityType: ServiceType.Alert,
|
|
1198
|
+
metricNames: Object.values(AlertMetricType),
|
|
1199
|
+
retentionDate: OneUptimeDate.addRemoveDays(
|
|
1200
|
+
OneUptimeDate.getCurrentDate(),
|
|
1201
|
+
metricRetentionDays,
|
|
1202
|
+
),
|
|
1203
|
+
});
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
return onDelete;
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1181
1211
|
@CaptureSpan()
|
|
1182
1212
|
public async changeAlertState(data: {
|
|
1183
1213
|
projectId: ObjectID;
|
|
@@ -1337,17 +1367,10 @@ ${alertSeverity.name}
|
|
|
1337
1367
|
alertStateTimelines[0];
|
|
1338
1368
|
|
|
1339
1369
|
/*
|
|
1340
|
-
* Serialize concurrent refreshes for this alert across pods.
|
|
1341
|
-
*
|
|
1342
|
-
*
|
|
1343
|
-
*
|
|
1344
|
-
* close-together transitions (e.g. auto-acknowledge then auto-resolve)
|
|
1345
|
-
* could interleave: both read a snapshot missing a metric and both
|
|
1346
|
-
* insert it. The Metric table is a plain MergeTree with no row collapse,
|
|
1347
|
-
* so those duplicates would be permanent and re-inflate the very Sum/Avg
|
|
1348
|
-
* widgets this change fixes. A best-effort distributed lock keyed on the
|
|
1349
|
-
* alert makes the read+insert atomic; if the lock can't be taken we still
|
|
1350
|
-
* proceed (the existence check alone covers the common sequential case).
|
|
1370
|
+
* Serialize concurrent refreshes for this alert across pods. Mutable
|
|
1371
|
+
* metrics are versioned inserts, but the replace operation also tombstones
|
|
1372
|
+
* stale metric-point identities. The lock keeps that read+insert cycle
|
|
1373
|
+
* ordered for each alert.
|
|
1351
1374
|
*/
|
|
1352
1375
|
let metricRefreshMutex: SemaphoreMutex | null = null;
|
|
1353
1376
|
try {
|
|
@@ -1367,48 +1390,7 @@ ${alertSeverity.name}
|
|
|
1367
1390
|
}
|
|
1368
1391
|
|
|
1369
1392
|
try {
|
|
1370
|
-
|
|
1371
|
-
* Write-once metrics — no deletes.
|
|
1372
|
-
*
|
|
1373
|
-
* The raw Metric table (`MetricItemV3`) feeds an AggregatingMergeTree
|
|
1374
|
-
* materialized view (`MetricItemAggMV1m_mv`) that accumulates
|
|
1375
|
-
* `sumState`/`countState` per (projectId, name, primaryEntityId,
|
|
1376
|
-
* minute). The MV trigger fires only on INSERT — an `ALTER ... DELETE`
|
|
1377
|
-
* mutation on the source rolls back nothing in the MV. So the old
|
|
1378
|
-
* "delete every metric for this alert, then re-insert" refresh both
|
|
1379
|
-
* (a) inflated those rollups and (b) issued one heavy `ALTER ... DELETE`
|
|
1380
|
-
* mutation per state transition per alert — a mutation storm.
|
|
1381
|
-
*
|
|
1382
|
-
* Every alert metric has a single eventually-final value: AlertCount is
|
|
1383
|
-
* constant `value = 1`; MTTA/MTTR/duration are each fixed once the
|
|
1384
|
-
* relevant state transition has happened. So we load what we've already
|
|
1385
|
-
* emitted for this alert and insert each metric exactly once, when it
|
|
1386
|
-
* becomes final. Repeat refreshes that find nothing new are no-ops.
|
|
1387
|
-
*/
|
|
1388
|
-
const existingMetricNames: Set<string> = new Set<string>(
|
|
1389
|
-
(
|
|
1390
|
-
await MetricService.findBy({
|
|
1391
|
-
query: {
|
|
1392
|
-
projectId: alert.projectId,
|
|
1393
|
-
primaryEntityId: data.alertId,
|
|
1394
|
-
},
|
|
1395
|
-
select: {
|
|
1396
|
-
name: true,
|
|
1397
|
-
},
|
|
1398
|
-
skip: 0,
|
|
1399
|
-
limit: LIMIT_PER_PROJECT,
|
|
1400
|
-
props: {
|
|
1401
|
-
isRoot: true,
|
|
1402
|
-
},
|
|
1403
|
-
})
|
|
1404
|
-
)
|
|
1405
|
-
.map((metric: Metric) => {
|
|
1406
|
-
return metric.name;
|
|
1407
|
-
})
|
|
1408
|
-
.filter(Boolean) as Array<string>,
|
|
1409
|
-
);
|
|
1410
|
-
|
|
1411
|
-
const itemsToSave: Array<Metric> = [];
|
|
1393
|
+
const itemsToSave: Array<MutableMetric> = [];
|
|
1412
1394
|
const metricTypesMap: Dictionary<MetricType> = {};
|
|
1413
1395
|
|
|
1414
1396
|
const metricRetentionDays: number = await this.getMetricRetentionDays();
|
|
@@ -1431,36 +1413,34 @@ ${alertSeverity.name}
|
|
|
1431
1413
|
alertCountMetricType.services = [];
|
|
1432
1414
|
metricTypesMap[AlertMetricType.AlertCount] = alertCountMetricType;
|
|
1433
1415
|
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
alertCountMetric.
|
|
1452
|
-
|
|
1453
|
-
);
|
|
1416
|
+
const alertCountMetric: MutableMetric = new MutableMetric();
|
|
1417
|
+
|
|
1418
|
+
alertCountMetric.projectId = alert.projectId;
|
|
1419
|
+
alertCountMetric.primaryEntityId = alert.id!;
|
|
1420
|
+
alertCountMetric.primaryEntityType = ServiceType.Alert;
|
|
1421
|
+
alertCountMetric.name = AlertMetricType.AlertCount;
|
|
1422
|
+
alertCountMetric.metricPointId = AlertMetricType.AlertCount;
|
|
1423
|
+
alertCountMetric.value = 1;
|
|
1424
|
+
alertCountMetric.attributes = {
|
|
1425
|
+
alertId: data.alertId.toString(),
|
|
1426
|
+
projectId: alert.projectId.toString(),
|
|
1427
|
+
monitorId: alert.monitor?._id?.toString(),
|
|
1428
|
+
monitorName: alert.monitor?.name?.toString(),
|
|
1429
|
+
alertSeverityId: alert.alertSeverity?._id?.toString(),
|
|
1430
|
+
alertSeverityName: alert.alertSeverity?.name?.toString(),
|
|
1431
|
+
};
|
|
1432
|
+
alertCountMetric.attributeKeys = TelemetryUtil.getAttributeKeys(
|
|
1433
|
+
alertCountMetric.attributes,
|
|
1434
|
+
);
|
|
1454
1435
|
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1436
|
+
alertCountMetric.time = alertStartsAt;
|
|
1437
|
+
alertCountMetric.timeUnixNano = OneUptimeDate.toUnixNano(
|
|
1438
|
+
alertCountMetric.time,
|
|
1439
|
+
);
|
|
1440
|
+
alertCountMetric.metricPointType = MetricPointType.Sum;
|
|
1441
|
+
alertCountMetric.retentionDate = alertMetricRetentionDate;
|
|
1461
1442
|
|
|
1462
|
-
|
|
1463
|
-
}
|
|
1443
|
+
itemsToSave.push(alertCountMetric);
|
|
1464
1444
|
|
|
1465
1445
|
// is the alert acknowledged?
|
|
1466
1446
|
const isAlertAcknowledged: boolean = alertStateTimelines.some(
|
|
@@ -1483,45 +1463,40 @@ ${alertSeverity.name}
|
|
|
1483
1463
|
metricType.unit = "seconds";
|
|
1484
1464
|
metricTypesMap[AlertMetricType.TimeToAcknowledge] = metricType;
|
|
1485
1465
|
|
|
1486
|
-
|
|
1487
|
-
if (!existingMetricNames.has(AlertMetricType.TimeToAcknowledge)) {
|
|
1488
|
-
const timeToAcknowledgeMetric: Metric = new Metric();
|
|
1489
|
-
|
|
1490
|
-
timeToAcknowledgeMetric.projectId = alert.projectId;
|
|
1491
|
-
timeToAcknowledgeMetric.primaryEntityId = alert.id!;
|
|
1492
|
-
timeToAcknowledgeMetric.primaryEntityType = ServiceType.Alert;
|
|
1493
|
-
timeToAcknowledgeMetric.name = AlertMetricType.TimeToAcknowledge;
|
|
1494
|
-
timeToAcknowledgeMetric.value =
|
|
1495
|
-
OneUptimeDate.getDifferenceInSeconds(
|
|
1496
|
-
ackAlertStateTimeline?.startsAt ||
|
|
1497
|
-
OneUptimeDate.getCurrentDate(),
|
|
1498
|
-
alertStartsAt,
|
|
1499
|
-
);
|
|
1500
|
-
timeToAcknowledgeMetric.attributes = {
|
|
1501
|
-
alertId: data.alertId.toString(),
|
|
1502
|
-
projectId: alert.projectId.toString(),
|
|
1503
|
-
monitorId: alert.monitor?._id?.toString(),
|
|
1504
|
-
monitorName: alert.monitor?.name?.toString(),
|
|
1505
|
-
alertSeverityId: alert.alertSeverity?._id?.toString(),
|
|
1506
|
-
alertSeverityName: alert.alertSeverity?.name?.toString(),
|
|
1507
|
-
};
|
|
1508
|
-
timeToAcknowledgeMetric.attributeKeys =
|
|
1509
|
-
TelemetryUtil.getAttributeKeys(
|
|
1510
|
-
timeToAcknowledgeMetric.attributes,
|
|
1511
|
-
);
|
|
1512
|
-
|
|
1513
|
-
timeToAcknowledgeMetric.time =
|
|
1514
|
-
ackAlertStateTimeline?.startsAt ||
|
|
1515
|
-
alert.createdAt ||
|
|
1516
|
-
OneUptimeDate.getCurrentDate();
|
|
1517
|
-
timeToAcknowledgeMetric.timeUnixNano = OneUptimeDate.toUnixNano(
|
|
1518
|
-
timeToAcknowledgeMetric.time,
|
|
1519
|
-
);
|
|
1520
|
-
timeToAcknowledgeMetric.metricPointType = MetricPointType.Sum;
|
|
1521
|
-
timeToAcknowledgeMetric.retentionDate = alertMetricRetentionDate;
|
|
1466
|
+
const timeToAcknowledgeMetric: MutableMetric = new MutableMetric();
|
|
1522
1467
|
|
|
1523
|
-
|
|
1524
|
-
|
|
1468
|
+
timeToAcknowledgeMetric.projectId = alert.projectId;
|
|
1469
|
+
timeToAcknowledgeMetric.primaryEntityId = alert.id!;
|
|
1470
|
+
timeToAcknowledgeMetric.primaryEntityType = ServiceType.Alert;
|
|
1471
|
+
timeToAcknowledgeMetric.name = AlertMetricType.TimeToAcknowledge;
|
|
1472
|
+
timeToAcknowledgeMetric.metricPointId =
|
|
1473
|
+
AlertMetricType.TimeToAcknowledge;
|
|
1474
|
+
timeToAcknowledgeMetric.value = OneUptimeDate.getDifferenceInSeconds(
|
|
1475
|
+
ackAlertStateTimeline?.startsAt || OneUptimeDate.getCurrentDate(),
|
|
1476
|
+
alertStartsAt,
|
|
1477
|
+
);
|
|
1478
|
+
timeToAcknowledgeMetric.attributes = {
|
|
1479
|
+
alertId: data.alertId.toString(),
|
|
1480
|
+
projectId: alert.projectId.toString(),
|
|
1481
|
+
monitorId: alert.monitor?._id?.toString(),
|
|
1482
|
+
monitorName: alert.monitor?.name?.toString(),
|
|
1483
|
+
alertSeverityId: alert.alertSeverity?._id?.toString(),
|
|
1484
|
+
alertSeverityName: alert.alertSeverity?.name?.toString(),
|
|
1485
|
+
};
|
|
1486
|
+
timeToAcknowledgeMetric.attributeKeys =
|
|
1487
|
+
TelemetryUtil.getAttributeKeys(timeToAcknowledgeMetric.attributes);
|
|
1488
|
+
|
|
1489
|
+
timeToAcknowledgeMetric.time =
|
|
1490
|
+
ackAlertStateTimeline?.startsAt ||
|
|
1491
|
+
alert.createdAt ||
|
|
1492
|
+
OneUptimeDate.getCurrentDate();
|
|
1493
|
+
timeToAcknowledgeMetric.timeUnixNano = OneUptimeDate.toUnixNano(
|
|
1494
|
+
timeToAcknowledgeMetric.time,
|
|
1495
|
+
);
|
|
1496
|
+
timeToAcknowledgeMetric.metricPointType = MetricPointType.Sum;
|
|
1497
|
+
timeToAcknowledgeMetric.retentionDate = alertMetricRetentionDate;
|
|
1498
|
+
|
|
1499
|
+
itemsToSave.push(timeToAcknowledgeMetric);
|
|
1525
1500
|
}
|
|
1526
1501
|
}
|
|
1527
1502
|
|
|
@@ -1545,54 +1520,43 @@ ${alertSeverity.name}
|
|
|
1545
1520
|
metricType.unit = "seconds";
|
|
1546
1521
|
metricTypesMap[AlertMetricType.TimeToResolve] = metricType;
|
|
1547
1522
|
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
timeToResolveMetric.
|
|
1570
|
-
|
|
1571
|
-
);
|
|
1523
|
+
const timeToResolveMetric: MutableMetric = new MutableMetric();
|
|
1524
|
+
|
|
1525
|
+
timeToResolveMetric.projectId = alert.projectId;
|
|
1526
|
+
timeToResolveMetric.primaryEntityId = alert.id!;
|
|
1527
|
+
timeToResolveMetric.primaryEntityType = ServiceType.Alert;
|
|
1528
|
+
timeToResolveMetric.name = AlertMetricType.TimeToResolve;
|
|
1529
|
+
timeToResolveMetric.metricPointId = AlertMetricType.TimeToResolve;
|
|
1530
|
+
timeToResolveMetric.value = OneUptimeDate.getDifferenceInSeconds(
|
|
1531
|
+
resolvedAlertStateTimeline?.startsAt ||
|
|
1532
|
+
OneUptimeDate.getCurrentDate(),
|
|
1533
|
+
alertStartsAt,
|
|
1534
|
+
);
|
|
1535
|
+
timeToResolveMetric.attributes = {
|
|
1536
|
+
alertId: data.alertId.toString(),
|
|
1537
|
+
projectId: alert.projectId.toString(),
|
|
1538
|
+
monitorId: alert.monitor?._id?.toString(),
|
|
1539
|
+
monitorName: alert.monitor?.name?.toString(),
|
|
1540
|
+
alertSeverityId: alert.alertSeverity?._id?.toString(),
|
|
1541
|
+
alertSeverityName: alert.alertSeverity?.name?.toString(),
|
|
1542
|
+
};
|
|
1543
|
+
timeToResolveMetric.attributeKeys = TelemetryUtil.getAttributeKeys(
|
|
1544
|
+
timeToResolveMetric.attributes,
|
|
1545
|
+
);
|
|
1572
1546
|
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1547
|
+
timeToResolveMetric.time =
|
|
1548
|
+
resolvedAlertStateTimeline?.startsAt ||
|
|
1549
|
+
alert.createdAt ||
|
|
1550
|
+
OneUptimeDate.getCurrentDate();
|
|
1551
|
+
timeToResolveMetric.timeUnixNano = OneUptimeDate.toUnixNano(
|
|
1552
|
+
timeToResolveMetric.time,
|
|
1553
|
+
);
|
|
1554
|
+
timeToResolveMetric.metricPointType = MetricPointType.Sum;
|
|
1555
|
+
timeToResolveMetric.retentionDate = alertMetricRetentionDate;
|
|
1582
1556
|
|
|
1583
|
-
|
|
1584
|
-
}
|
|
1557
|
+
itemsToSave.push(timeToResolveMetric);
|
|
1585
1558
|
}
|
|
1586
1559
|
|
|
1587
|
-
/*
|
|
1588
|
-
* Alert duration — write-once, finalized at resolution.
|
|
1589
|
-
*
|
|
1590
|
-
* Previously recomputed as (latest state transition − start) on every
|
|
1591
|
-
* refresh, which grew for open alerts and required delete + re-insert.
|
|
1592
|
-
* With no deletes we emit a single final duration once the alert reaches
|
|
1593
|
-
* a resolved state: (first resolution − start). An alert that is never
|
|
1594
|
-
* resolved has no duration row (its lifetime is not yet final).
|
|
1595
|
-
*/
|
|
1596
1560
|
if (isAlertResolved && resolvedAlertStateTimeline) {
|
|
1597
1561
|
// register the metric type so the catalog stays complete across refreshes.
|
|
1598
1562
|
const metricType: MetricType = new MetricType();
|
|
@@ -1601,53 +1565,50 @@ ${alertSeverity.name}
|
|
|
1601
1565
|
metricType.unit = "seconds";
|
|
1602
1566
|
metricTypesMap[AlertMetricType.AlertDuration] = metricType;
|
|
1603
1567
|
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
resolvedAlertStateTimeline.startsAt ||
|
|
1607
|
-
OneUptimeDate.getCurrentDate();
|
|
1568
|
+
const alertEndsAt: Date =
|
|
1569
|
+
resolvedAlertStateTimeline.startsAt || OneUptimeDate.getCurrentDate();
|
|
1608
1570
|
|
|
1609
|
-
|
|
1571
|
+
const alertDurationMetric: MutableMetric = new MutableMetric();
|
|
1610
1572
|
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1573
|
+
alertDurationMetric.projectId = alert.projectId;
|
|
1574
|
+
alertDurationMetric.primaryEntityId = alert.id!;
|
|
1575
|
+
alertDurationMetric.primaryEntityType = ServiceType.Alert;
|
|
1576
|
+
alertDurationMetric.name = AlertMetricType.AlertDuration;
|
|
1577
|
+
alertDurationMetric.metricPointId = AlertMetricType.AlertDuration;
|
|
1578
|
+
alertDurationMetric.value = OneUptimeDate.getDifferenceInSeconds(
|
|
1579
|
+
alertEndsAt,
|
|
1580
|
+
alertStartsAt,
|
|
1581
|
+
);
|
|
1582
|
+
alertDurationMetric.attributes = {
|
|
1583
|
+
alertId: data.alertId.toString(),
|
|
1584
|
+
projectId: alert.projectId.toString(),
|
|
1585
|
+
monitorId: alert.monitor?._id?.toString(),
|
|
1586
|
+
monitorName: alert.monitor?.name?.toString(),
|
|
1587
|
+
alertSeverityId: alert.alertSeverity?._id?.toString(),
|
|
1588
|
+
alertSeverityName: alert.alertSeverity?.name?.toString(),
|
|
1589
|
+
};
|
|
1590
|
+
alertDurationMetric.attributeKeys = TelemetryUtil.getAttributeKeys(
|
|
1591
|
+
alertDurationMetric.attributes,
|
|
1592
|
+
);
|
|
1630
1593
|
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1594
|
+
alertDurationMetric.time = alertEndsAt;
|
|
1595
|
+
alertDurationMetric.timeUnixNano = OneUptimeDate.toUnixNano(
|
|
1596
|
+
alertDurationMetric.time,
|
|
1597
|
+
);
|
|
1598
|
+
alertDurationMetric.metricPointType = MetricPointType.Sum;
|
|
1599
|
+
alertDurationMetric.retentionDate = alertMetricRetentionDate;
|
|
1637
1600
|
|
|
1638
|
-
|
|
1639
|
-
}
|
|
1601
|
+
itemsToSave.push(alertDurationMetric);
|
|
1640
1602
|
}
|
|
1641
1603
|
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
}
|
|
1604
|
+
await MutableMetricService.replaceEntityMetrics({
|
|
1605
|
+
projectId: alert.projectId,
|
|
1606
|
+
primaryEntityId: alert.id!,
|
|
1607
|
+
primaryEntityType: ServiceType.Alert,
|
|
1608
|
+
metricNames: Object.values(AlertMetricType),
|
|
1609
|
+
metrics: itemsToSave,
|
|
1610
|
+
retentionDate: alertMetricRetentionDate,
|
|
1611
|
+
});
|
|
1651
1612
|
|
|
1652
1613
|
TelemetryUtil.indexMetricNameServiceNameMap({
|
|
1653
1614
|
metricNameServiceNameMap: metricTypesMap,
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import CreateBy from "../Types/Database/CreateBy";
|
|
2
2
|
import DeleteBy from "../Types/Database/DeleteBy";
|
|
3
|
-
import { OnCreate, OnDelete } from "../Types/Database/Hooks";
|
|
3
|
+
import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
|
|
4
4
|
import QueryHelper from "../Types/Database/QueryHelper";
|
|
5
|
+
import UpdateBy from "../Types/Database/UpdateBy";
|
|
5
6
|
import DatabaseService from "./DatabaseService";
|
|
6
7
|
import AlertService from "./AlertService";
|
|
7
8
|
import AlertStateService from "./AlertStateService";
|
|
@@ -505,6 +506,111 @@ ${createdItem.rootCause}`,
|
|
|
505
506
|
return createdItem;
|
|
506
507
|
}
|
|
507
508
|
|
|
509
|
+
@CaptureSpan()
|
|
510
|
+
protected override async onBeforeUpdate(
|
|
511
|
+
updateBy: UpdateBy<AlertStateTimeline>,
|
|
512
|
+
): Promise<OnUpdate<AlertStateTimeline>> {
|
|
513
|
+
const alertIds: Array<ObjectID> =
|
|
514
|
+
await this.getAlertIdsForTimelineQuery(updateBy);
|
|
515
|
+
|
|
516
|
+
return {
|
|
517
|
+
updateBy,
|
|
518
|
+
carryForward: alertIds,
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
@CaptureSpan()
|
|
523
|
+
protected override async onUpdateSuccess(
|
|
524
|
+
onUpdate: OnUpdate<AlertStateTimeline>,
|
|
525
|
+
updatedItemIds: ObjectID[],
|
|
526
|
+
): Promise<OnUpdate<AlertStateTimeline>> {
|
|
527
|
+
const alertIds: Array<ObjectID> = [
|
|
528
|
+
...(Array.isArray(onUpdate.carryForward)
|
|
529
|
+
? (onUpdate.carryForward as Array<ObjectID>)
|
|
530
|
+
: []),
|
|
531
|
+
];
|
|
532
|
+
|
|
533
|
+
if (updatedItemIds.length > 0) {
|
|
534
|
+
const updatedTimelines: Array<AlertStateTimeline> = await this.findBy({
|
|
535
|
+
query: {
|
|
536
|
+
_id: QueryHelper.any(updatedItemIds),
|
|
537
|
+
},
|
|
538
|
+
select: {
|
|
539
|
+
alertId: true,
|
|
540
|
+
},
|
|
541
|
+
skip: 0,
|
|
542
|
+
limit: LIMIT_PER_PROJECT,
|
|
543
|
+
props: {
|
|
544
|
+
isRoot: true,
|
|
545
|
+
ignoreHooks: true,
|
|
546
|
+
},
|
|
547
|
+
});
|
|
548
|
+
|
|
549
|
+
alertIds.push(...this.getAlertIdsFromTimelines(updatedTimelines));
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
this.refreshAlertMetricsForAlertIds(alertIds);
|
|
553
|
+
|
|
554
|
+
return onUpdate;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
private async getAlertIdsForTimelineQuery(
|
|
558
|
+
updateBy: UpdateBy<AlertStateTimeline>,
|
|
559
|
+
): Promise<Array<ObjectID>> {
|
|
560
|
+
const timelines: Array<AlertStateTimeline> = await this.findBy({
|
|
561
|
+
query: updateBy.query,
|
|
562
|
+
select: {
|
|
563
|
+
alertId: true,
|
|
564
|
+
},
|
|
565
|
+
skip: updateBy.skip,
|
|
566
|
+
limit: updateBy.limit,
|
|
567
|
+
props: {
|
|
568
|
+
isRoot: true,
|
|
569
|
+
ignoreHooks: true,
|
|
570
|
+
},
|
|
571
|
+
});
|
|
572
|
+
|
|
573
|
+
return this.getAlertIdsFromTimelines(timelines);
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
private getAlertIdsFromTimelines(
|
|
577
|
+
timelines: Array<AlertStateTimeline>,
|
|
578
|
+
): Array<ObjectID> {
|
|
579
|
+
const alertIds: Map<string, ObjectID> = new Map<string, ObjectID>();
|
|
580
|
+
|
|
581
|
+
for (const timeline of timelines) {
|
|
582
|
+
if (timeline.alertId) {
|
|
583
|
+
alertIds.set(timeline.alertId.toString(), timeline.alertId);
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
return Array.from(alertIds.values());
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
private refreshAlertMetricsForAlertIds(alertIds: Array<ObjectID>): void {
|
|
591
|
+
const uniqueAlertIds: Map<string, ObjectID> = new Map<string, ObjectID>();
|
|
592
|
+
|
|
593
|
+
for (const alertId of alertIds) {
|
|
594
|
+
uniqueAlertIds.set(alertId.toString(), alertId);
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
for (const alertId of uniqueAlertIds.values()) {
|
|
598
|
+
AlertService.refreshAlertMetrics({
|
|
599
|
+
alertId: alertId,
|
|
600
|
+
}).catch((error: Error) => {
|
|
601
|
+
logger.error(
|
|
602
|
+
"Error while refreshing alert metrics after alert state timeline update",
|
|
603
|
+
{
|
|
604
|
+
alertId: alertId.toString(),
|
|
605
|
+
} as LogAttributes,
|
|
606
|
+
);
|
|
607
|
+
logger.error(error, {
|
|
608
|
+
alertId: alertId.toString(),
|
|
609
|
+
} as LogAttributes);
|
|
610
|
+
});
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
|
|
508
614
|
private async isLastAlertState(data: {
|
|
509
615
|
projectId: ObjectID;
|
|
510
616
|
alertStateId: ObjectID;
|
|
@@ -727,6 +833,20 @@ ${createdItem.rootCause}`,
|
|
|
727
833
|
},
|
|
728
834
|
});
|
|
729
835
|
}
|
|
836
|
+
|
|
837
|
+
AlertService.refreshAlertMetrics({
|
|
838
|
+
alertId: alertId,
|
|
839
|
+
}).catch((error: Error) => {
|
|
840
|
+
logger.error(
|
|
841
|
+
"Error while refreshing alert metrics after alert state timeline deletion",
|
|
842
|
+
{
|
|
843
|
+
alertId: alertId?.toString(),
|
|
844
|
+
} as LogAttributes,
|
|
845
|
+
);
|
|
846
|
+
logger.error(error, {
|
|
847
|
+
alertId: alertId?.toString(),
|
|
848
|
+
} as LogAttributes);
|
|
849
|
+
});
|
|
730
850
|
}
|
|
731
851
|
|
|
732
852
|
return onDelete;
|
|
@@ -1953,34 +1953,30 @@ export default class AnalyticsDatabaseService<
|
|
|
1953
1953
|
|
|
1954
1954
|
for (const columns of data.getRequiredColumns()) {
|
|
1955
1955
|
const requiredField: string = columns.key;
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
) {
|
|
1956
|
+
const value: unknown = (data as any)[requiredField];
|
|
1957
|
+
|
|
1958
|
+
/*
|
|
1959
|
+
* A present value satisfies the requirement — including `false`, `0` and
|
|
1960
|
+
* `""`, which are all valid column values. Only a genuinely absent value
|
|
1961
|
+
* (null/undefined) triggers a default-fill or a validation error.
|
|
1962
|
+
*
|
|
1963
|
+
* This deliberately treats booleans like every other type. The previous
|
|
1964
|
+
* implementation special-cased booleans with `!val && val !== false`,
|
|
1965
|
+
* which can never be true for a real boolean, so it threw for EVERY
|
|
1966
|
+
* required boolean (both true and false) — making any required Boolean
|
|
1967
|
+
* column impossible to insert via createMany.
|
|
1968
|
+
*/
|
|
1969
|
+
if (value !== null && value !== undefined) {
|
|
1970
|
+
continue;
|
|
1971
|
+
}
|
|
1972
|
+
|
|
1973
|
+
if (data.isDefaultValueColumn(requiredField)) {
|
|
1974
1974
|
// add default value.
|
|
1975
1975
|
data.setColumnValue(
|
|
1976
1976
|
requiredField,
|
|
1977
1977
|
data.getDefaultValueForColumn(requiredField),
|
|
1978
1978
|
);
|
|
1979
|
-
} else
|
|
1980
|
-
((data as any)[requiredField] === null ||
|
|
1981
|
-
(data as any)[requiredField] === undefined) &&
|
|
1982
|
-
!data.isDefaultValueColumn(requiredField)
|
|
1983
|
-
) {
|
|
1979
|
+
} else {
|
|
1984
1980
|
throw new BadDataException(`${requiredField} is required`);
|
|
1985
1981
|
}
|
|
1986
1982
|
}
|