@oneuptime/common 10.0.71 → 10.0.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Models/DatabaseModels/Alert.ts +55 -0
- package/Models/DatabaseModels/Incident.ts +55 -0
- package/Models/DatabaseModels/StatusPage.ts +80 -0
- package/Server/API/StatusPageAPI.ts +4 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1776940714709-MigrationName.ts +41 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/1776971364783-AddStatusPageLanguageSettings.ts +25 -0
- package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +4 -0
- package/Server/Services/AnalyticsDatabaseService.ts +17 -7
- package/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.ts +175 -29
- package/Server/Utils/Monitor/Criteria/ServerMonitorCriteria.ts +71 -0
- package/Server/Utils/Monitor/MonitorAlert.ts +91 -7
- package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts +171 -2
- package/Server/Utils/Monitor/MonitorIncident.ts +133 -8
- package/Server/Utils/Monitor/MonitorMetricUtil.ts +423 -1
- package/Server/Utils/Monitor/MonitorResource.ts +2 -0
- package/Server/Utils/Monitor/MonitorTemplateUtil.ts +99 -0
- package/Tests/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.test.ts +268 -0
- package/Types/Infrastructure/BasicMetrics.ts +75 -0
- package/Types/Metrics/MetricQueryData.ts +11 -0
- package/Types/Monitor/CriteriaFilter.ts +10 -0
- package/Types/Monitor/MetricMonitor/MetricCriteriaContext.ts +11 -0
- package/Types/Monitor/MetricMonitor/MetricMonitorResponse.ts +10 -0
- package/Types/Monitor/MetricMonitor/MetricSeriesResult.ts +20 -0
- package/Types/Monitor/MonitorMetricType.ts +34 -0
- package/Types/Monitor/ServerMonitor/ServerMonitorResponse.ts +8 -0
- package/Types/Probe/ProbeApiIngestResponse.ts +25 -0
- package/Types/StatusPage/StatusPageLanguage.ts +29 -0
- package/UI/Components/Charts/Area/AreaChart.tsx +17 -12
- package/UI/Components/Charts/Bar/BarChart.tsx +16 -11
- package/UI/Components/Charts/ChartGroup/ChartGroup.tsx +23 -0
- package/UI/Components/Charts/Line/LineChart.tsx +16 -11
- package/UI/Components/Filters/FiltersForm.tsx +26 -2
- package/UI/Components/MonitorTemplateVariables/TemplateVariablesCatalog.ts +453 -0
- package/UI/Components/MonitorTemplateVariables/TemplateVariablesModal.tsx +229 -0
- package/Utils/Metrics/MetricSeriesFingerprint.ts +97 -0
- package/Utils/Monitor/MonitorMetricType.ts +309 -19
- package/build/dist/Models/DatabaseModels/Alert.js +57 -0
- package/build/dist/Models/DatabaseModels/Alert.js.map +1 -1
- package/build/dist/Models/DatabaseModels/Incident.js +57 -0
- package/build/dist/Models/DatabaseModels/Incident.js.map +1 -1
- package/build/dist/Models/DatabaseModels/StatusPage.js +82 -0
- package/build/dist/Models/DatabaseModels/StatusPage.js.map +1 -1
- package/build/dist/Server/API/StatusPageAPI.js +4 -0
- package/build/dist/Server/API/StatusPageAPI.js.map +1 -1
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1776940714709-MigrationName.js +22 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1776940714709-MigrationName.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1776971364783-AddStatusPageLanguageSettings.js +14 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1776971364783-AddStatusPageLanguageSettings.js.map +1 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +4 -0
- package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
- package/build/dist/Server/Services/AnalyticsDatabaseService.js +14 -4
- package/build/dist/Server/Services/AnalyticsDatabaseService.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.js +132 -30
- package/build/dist/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/Criteria/ServerMonitorCriteria.js +58 -7
- package/build/dist/Server/Utils/Monitor/Criteria/ServerMonitorCriteria.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorAlert.js +66 -12
- package/build/dist/Server/Utils/Monitor/MonitorAlert.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorCriteriaEvaluator.js +112 -0
- package/build/dist/Server/Utils/Monitor/MonitorCriteriaEvaluator.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorIncident.js +91 -15
- package/build/dist/Server/Utils/Monitor/MonitorIncident.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorMetricUtil.js +373 -0
- package/build/dist/Server/Utils/Monitor/MonitorMetricUtil.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorResource.js +2 -0
- package/build/dist/Server/Utils/Monitor/MonitorResource.js.map +1 -1
- package/build/dist/Server/Utils/Monitor/MonitorTemplateUtil.js +65 -0
- package/build/dist/Server/Utils/Monitor/MonitorTemplateUtil.js.map +1 -1
- package/build/dist/Tests/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.test.js +199 -0
- package/build/dist/Tests/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.test.js.map +1 -1
- package/build/dist/Types/Monitor/CriteriaFilter.js +10 -0
- package/build/dist/Types/Monitor/CriteriaFilter.js.map +1 -1
- package/build/dist/Types/Monitor/MetricMonitor/MetricSeriesResult.js +2 -0
- package/build/dist/Types/Monitor/MetricMonitor/MetricSeriesResult.js.map +1 -0
- package/build/dist/Types/Monitor/MonitorMetricType.js +28 -0
- package/build/dist/Types/Monitor/MonitorMetricType.js.map +1 -1
- package/build/dist/Types/StatusPage/StatusPageLanguage.js +21 -0
- package/build/dist/Types/StatusPage/StatusPageLanguage.js.map +1 -0
- package/build/dist/UI/Components/Charts/Area/AreaChart.js +13 -12
- package/build/dist/UI/Components/Charts/Area/AreaChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/Bar/BarChart.js +12 -11
- package/build/dist/UI/Components/Charts/Bar/BarChart.js.map +1 -1
- package/build/dist/UI/Components/Charts/ChartGroup/ChartGroup.js +11 -3
- package/build/dist/UI/Components/Charts/ChartGroup/ChartGroup.js.map +1 -1
- package/build/dist/UI/Components/Charts/Line/LineChart.js +12 -11
- package/build/dist/UI/Components/Charts/Line/LineChart.js.map +1 -1
- package/build/dist/UI/Components/Filters/FiltersForm.js +6 -2
- package/build/dist/UI/Components/Filters/FiltersForm.js.map +1 -1
- package/build/dist/UI/Components/MonitorTemplateVariables/TemplateVariablesCatalog.js +383 -0
- package/build/dist/UI/Components/MonitorTemplateVariables/TemplateVariablesCatalog.js.map +1 -0
- package/build/dist/UI/Components/MonitorTemplateVariables/TemplateVariablesModal.js +109 -0
- package/build/dist/UI/Components/MonitorTemplateVariables/TemplateVariablesModal.js.map +1 -0
- package/build/dist/Utils/Metrics/MetricSeriesFingerprint.js +81 -0
- package/build/dist/Utils/Metrics/MetricSeriesFingerprint.js.map +1 -0
- package/build/dist/Utils/Monitor/MonitorMetricType.js +287 -19
- package/build/dist/Utils/Monitor/MonitorMetricType.js.map +1 -1
- package/package.json +1 -1
|
@@ -29,6 +29,7 @@ import { JSONObject } from "../../../Types/JSON";
|
|
|
29
29
|
import OneUptimeDate from "../../../Types/Date";
|
|
30
30
|
import MonitorEvaluationSummary from "../../../Types/Monitor/MonitorEvaluationSummary";
|
|
31
31
|
import { IncidentMemberRoleAssignment } from "../../../Types/Monitor/CriteriaIncident";
|
|
32
|
+
import { PerSeriesCriteriaMatch } from "../../../Types/Probe/ProbeApiIngestResponse";
|
|
32
33
|
|
|
33
34
|
export default class MonitorIncident {
|
|
34
35
|
@CaptureSpan()
|
|
@@ -41,6 +42,15 @@ export default class MonitorIncident {
|
|
|
41
42
|
criteriaInstance: MonitorCriteriaInstance | null;
|
|
42
43
|
dataToProcess: DataToProcess;
|
|
43
44
|
evaluationSummary?: MonitorEvaluationSummary | undefined;
|
|
45
|
+
/**
|
|
46
|
+
* When set, the fingerprint set of series still breaching on this
|
|
47
|
+
* tick. Any open per-series incident whose fingerprint is NOT in
|
|
48
|
+
* this set is auto-resolved — that's how a series returning to
|
|
49
|
+
* normal closes its incident independently of other series on the
|
|
50
|
+
* same monitor. Undefined means "legacy mode" and per-series
|
|
51
|
+
* incidents are treated like any other for dedupe/resolve.
|
|
52
|
+
*/
|
|
53
|
+
breachingSeriesFingerprints?: Set<string> | undefined;
|
|
44
54
|
}): Promise<Array<Incident>> {
|
|
45
55
|
// check active incidents and if there are open incidents, do not cretae anothr incident.
|
|
46
56
|
const openIncidents: Array<Incident> = await IncidentService.findBy({
|
|
@@ -54,12 +64,15 @@ export default class MonitorIncident {
|
|
|
54
64
|
limit: LIMIT_PER_PROJECT,
|
|
55
65
|
select: {
|
|
56
66
|
_id: true,
|
|
67
|
+
title: true,
|
|
57
68
|
createdCriteriaId: true,
|
|
58
69
|
createdIncidentTemplateId: true,
|
|
59
70
|
projectId: true,
|
|
60
71
|
incidentNumber: true,
|
|
61
72
|
incidentNumberWithPrefix: true,
|
|
62
73
|
currentIncidentStateId: true,
|
|
74
|
+
seriesFingerprint: true,
|
|
75
|
+
seriesLabels: true,
|
|
63
76
|
},
|
|
64
77
|
props: {
|
|
65
78
|
isRoot: true,
|
|
@@ -74,6 +87,7 @@ export default class MonitorIncident {
|
|
|
74
87
|
autoResolveCriteriaInstanceIdIncidentIdsDictionary:
|
|
75
88
|
input.autoResolveCriteriaInstanceIdIncidentIdsDictionary,
|
|
76
89
|
criteriaInstance: input.criteriaInstance,
|
|
90
|
+
breachingSeriesFingerprints: input.breachingSeriesFingerprints,
|
|
77
91
|
});
|
|
78
92
|
|
|
79
93
|
if (shouldClose) {
|
|
@@ -115,6 +129,13 @@ export default class MonitorIncident {
|
|
|
115
129
|
props: {
|
|
116
130
|
telemetryQuery?: TelemetryQuery | undefined;
|
|
117
131
|
};
|
|
132
|
+
/**
|
|
133
|
+
* When set, create one incident per series instead of one per
|
|
134
|
+
* monitor. Each entry gets its own rootCause, seriesFingerprint,
|
|
135
|
+
* and seriesLabels so the incident title + description can
|
|
136
|
+
* reference `{{host.name}}` etc. via the template engine.
|
|
137
|
+
*/
|
|
138
|
+
matchesPerSeries?: Array<PerSeriesCriteriaMatch> | undefined;
|
|
118
139
|
}): Promise<void> {
|
|
119
140
|
const incidentLogAttributes: LogAttributes = {
|
|
120
141
|
projectId: input.monitor.projectId?.toString(),
|
|
@@ -125,6 +146,21 @@ export default class MonitorIncident {
|
|
|
125
146
|
`${input.monitor.id?.toString()} - Check open incidents.`,
|
|
126
147
|
incidentLogAttributes,
|
|
127
148
|
);
|
|
149
|
+
|
|
150
|
+
/*
|
|
151
|
+
* Per-series mode: close any open incident for a series that's no
|
|
152
|
+
* longer breaching *before* we look at the remaining open set, so
|
|
153
|
+
* dedupe decisions below match the post-resolve state.
|
|
154
|
+
*/
|
|
155
|
+
const breachingSeriesFingerprints: Set<string> | undefined =
|
|
156
|
+
input.matchesPerSeries
|
|
157
|
+
? new Set<string>(
|
|
158
|
+
input.matchesPerSeries.map((m: PerSeriesCriteriaMatch) => {
|
|
159
|
+
return m.fingerprint;
|
|
160
|
+
}),
|
|
161
|
+
)
|
|
162
|
+
: undefined;
|
|
163
|
+
|
|
128
164
|
// check active incidents and if there are open incidents, do not cretae anothr incident.
|
|
129
165
|
const openIncidents: Array<Incident> =
|
|
130
166
|
await this.checkOpenIncidentsAndCloseIfResolved({
|
|
@@ -135,14 +171,29 @@ export default class MonitorIncident {
|
|
|
135
171
|
criteriaInstance: input.criteriaInstance,
|
|
136
172
|
dataToProcess: input.dataToProcess,
|
|
137
173
|
evaluationSummary: input.evaluationSummary,
|
|
174
|
+
breachingSeriesFingerprints,
|
|
138
175
|
});
|
|
139
176
|
|
|
140
|
-
if (input.criteriaInstance.data?.createIncidents) {
|
|
141
|
-
|
|
177
|
+
if (!input.criteriaInstance.data?.createIncidents) {
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
142
180
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
181
|
+
/*
|
|
182
|
+
* Series-less path: one incident per criteriaIncident template as
|
|
183
|
+
* before. Series-aware path: one incident per (series × template).
|
|
184
|
+
*/
|
|
185
|
+
const seriesToProcess: Array<PerSeriesCriteriaMatch | undefined> =
|
|
186
|
+
input.matchesPerSeries && input.matchesPerSeries.length > 0
|
|
187
|
+
? input.matchesPerSeries
|
|
188
|
+
: [undefined];
|
|
189
|
+
|
|
190
|
+
for (const criteriaIncident of input.criteriaInstance.data?.incidents ||
|
|
191
|
+
[]) {
|
|
192
|
+
for (const seriesMatch of seriesToProcess) {
|
|
193
|
+
const seriesFingerprint: string | undefined = seriesMatch?.fingerprint;
|
|
194
|
+
const seriesLabels: JSONObject | undefined = seriesMatch?.labels;
|
|
195
|
+
const seriesRootCause: string =
|
|
196
|
+
seriesMatch?.rootCause || input.rootCause;
|
|
146
197
|
|
|
147
198
|
const alreadyOpenIncident: Incident | undefined = openIncidents.find(
|
|
148
199
|
(incident: Incident) => {
|
|
@@ -150,7 +201,8 @@ export default class MonitorIncident {
|
|
|
150
201
|
incident.createdCriteriaId ===
|
|
151
202
|
input.criteriaInstance.data?.id.toString() &&
|
|
152
203
|
incident.createdIncidentTemplateId ===
|
|
153
|
-
criteriaIncident.id.toString()
|
|
204
|
+
criteriaIncident.id.toString() &&
|
|
205
|
+
(incident.seriesFingerprint || undefined) === seriesFingerprint
|
|
154
206
|
);
|
|
155
207
|
},
|
|
156
208
|
);
|
|
@@ -168,9 +220,20 @@ export default class MonitorIncident {
|
|
|
168
220
|
);
|
|
169
221
|
|
|
170
222
|
if (hasAlreadyOpenIncident) {
|
|
223
|
+
/*
|
|
224
|
+
* Use the open incident's already-rendered title when
|
|
225
|
+
* available — the template (`criteriaIncident.title`) still
|
|
226
|
+
* contains unresolved `{{…}}` placeholders because it's the
|
|
227
|
+
* criterion's template string, not the instance's rendered
|
|
228
|
+
* output. Falling back to the template only when the open
|
|
229
|
+
* incident somehow has no title.
|
|
230
|
+
*/
|
|
231
|
+
const renderedTitle: string =
|
|
232
|
+
alreadyOpenIncident?.title || criteriaIncident.title;
|
|
233
|
+
|
|
171
234
|
input.evaluationSummary?.events.push({
|
|
172
235
|
type: "incident-skipped",
|
|
173
|
-
title: `Incident already active: ${
|
|
236
|
+
title: `Incident already active: ${renderedTitle}`,
|
|
174
237
|
message:
|
|
175
238
|
"Skipped creating a new incident because an active incident exists for this criteria.",
|
|
176
239
|
relatedCriteriaId: input.criteriaInstance.data?.id,
|
|
@@ -193,6 +256,8 @@ export default class MonitorIncident {
|
|
|
193
256
|
MonitorTemplateUtil.buildTemplateStorageMap({
|
|
194
257
|
monitorType: input.monitor.monitorType!,
|
|
195
258
|
dataToProcess: input.dataToProcess,
|
|
259
|
+
monitor: input.monitor,
|
|
260
|
+
seriesLabels,
|
|
196
261
|
});
|
|
197
262
|
|
|
198
263
|
incident.title = MonitorTemplateUtil.processTemplateString({
|
|
@@ -236,7 +301,7 @@ export default class MonitorIncident {
|
|
|
236
301
|
|
|
237
302
|
incident.monitors = [input.monitor];
|
|
238
303
|
incident.projectId = input.monitor.projectId!;
|
|
239
|
-
incident.rootCause =
|
|
304
|
+
incident.rootCause = seriesRootCause;
|
|
240
305
|
incident.createdStateLog = JSON.parse(
|
|
241
306
|
JSON.stringify(input.dataToProcess, null, 2),
|
|
242
307
|
);
|
|
@@ -245,6 +310,13 @@ export default class MonitorIncident {
|
|
|
245
310
|
|
|
246
311
|
incident.createdIncidentTemplateId = criteriaIncident.id.toString();
|
|
247
312
|
|
|
313
|
+
if (seriesFingerprint) {
|
|
314
|
+
incident.seriesFingerprint = seriesFingerprint;
|
|
315
|
+
}
|
|
316
|
+
if (seriesLabels && Object.keys(seriesLabels).length > 0) {
|
|
317
|
+
incident.seriesLabels = seriesLabels;
|
|
318
|
+
}
|
|
319
|
+
|
|
248
320
|
incident.onCallDutyPolicies =
|
|
249
321
|
criteriaIncident.onCallPolicyIds?.map((id: ObjectID) => {
|
|
250
322
|
const onCallPolicy: OnCallDutyPolicy = new OnCallDutyPolicy();
|
|
@@ -504,7 +576,60 @@ export default class MonitorIncident {
|
|
|
504
576
|
Array<string>
|
|
505
577
|
>;
|
|
506
578
|
criteriaInstance: MonitorCriteriaInstance | null; // null if no criteia met.
|
|
579
|
+
breachingSeriesFingerprints?: Set<string> | undefined;
|
|
507
580
|
}): boolean {
|
|
581
|
+
const openSeriesFingerprint: string | undefined =
|
|
582
|
+
input.openIncident.seriesFingerprint || undefined;
|
|
583
|
+
|
|
584
|
+
/*
|
|
585
|
+
* Per-series auto-resolve: when the monitor emits a breaching-
|
|
586
|
+
* series set and this open incident has a fingerprint, resolve
|
|
587
|
+
* whenever the fingerprint is no longer in the set — regardless
|
|
588
|
+
* of whether some *other* series is still breaching the same
|
|
589
|
+
* criteria. This is the whole point of per-host incidents.
|
|
590
|
+
*/
|
|
591
|
+
if (
|
|
592
|
+
input.breachingSeriesFingerprints !== undefined &&
|
|
593
|
+
openSeriesFingerprint
|
|
594
|
+
) {
|
|
595
|
+
const stillBreaching: boolean = input.breachingSeriesFingerprints.has(
|
|
596
|
+
openSeriesFingerprint,
|
|
597
|
+
);
|
|
598
|
+
|
|
599
|
+
if (stillBreaching) {
|
|
600
|
+
return false;
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
/*
|
|
604
|
+
* Series no longer breaching. Only auto-close if the criteria
|
|
605
|
+
* was configured to auto-resolve in the first place; otherwise
|
|
606
|
+
* stay open so a human can acknowledge.
|
|
607
|
+
*/
|
|
608
|
+
if (!input.openIncident.createdCriteriaId?.toString()) {
|
|
609
|
+
return false;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
if (!input.openIncident.createdIncidentTemplateId?.toString()) {
|
|
613
|
+
return false;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
const autoResolveTemplates: Array<string> | undefined =
|
|
617
|
+
input.autoResolveCriteriaInstanceIdIncidentIdsDictionary[
|
|
618
|
+
input.openIncident.createdCriteriaId.toString()
|
|
619
|
+
];
|
|
620
|
+
|
|
621
|
+
if (
|
|
622
|
+
autoResolveTemplates &&
|
|
623
|
+
autoResolveTemplates.includes(
|
|
624
|
+
input.openIncident.createdIncidentTemplateId.toString(),
|
|
625
|
+
)
|
|
626
|
+
) {
|
|
627
|
+
return true;
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
return false;
|
|
631
|
+
}
|
|
632
|
+
|
|
508
633
|
if (
|
|
509
634
|
input.openIncident.createdCriteriaId?.toString() ===
|
|
510
635
|
input.criteriaInstance?.data?.id.toString()
|
|
@@ -10,7 +10,9 @@ import {
|
|
|
10
10
|
ServiceType,
|
|
11
11
|
} from "../../../Models/AnalyticsModels/Metric";
|
|
12
12
|
import MetricType from "../../../Models/DatabaseModels/MetricType";
|
|
13
|
-
import BasicInfrastructureMetrics
|
|
13
|
+
import BasicInfrastructureMetrics, {
|
|
14
|
+
NetworkInterfaceMetrics,
|
|
15
|
+
} from "../../../Types/Infrastructure/BasicMetrics";
|
|
14
16
|
import Dictionary from "../../../Types/Dictionary";
|
|
15
17
|
import { JSONObject } from "../../../Types/JSON";
|
|
16
18
|
import CapturedMetric from "../../../Types/Monitor/CustomCodeMonitor/CapturedMetric";
|
|
@@ -159,6 +161,73 @@ export default class MonitorMetricUtil {
|
|
|
159
161
|
} as JSONObject;
|
|
160
162
|
}
|
|
161
163
|
|
|
164
|
+
/*
|
|
165
|
+
* Helper that collapses the "build attributes → build row → push → register MetricType"
|
|
166
|
+
* pattern used repeatedly below. Silently skips emission when value is missing/non-finite
|
|
167
|
+
* so callers can pass optional agent fields directly.
|
|
168
|
+
*/
|
|
169
|
+
private static async pushMonitorMetric(data: {
|
|
170
|
+
projectId: ObjectID;
|
|
171
|
+
monitorId: ObjectID;
|
|
172
|
+
monitorName: string | undefined;
|
|
173
|
+
probeName: string | undefined;
|
|
174
|
+
metricName: string;
|
|
175
|
+
value: number | null | undefined;
|
|
176
|
+
description: string;
|
|
177
|
+
unit: string;
|
|
178
|
+
extraAttributes?: JSONObject;
|
|
179
|
+
metricPointType?: MetricPointType;
|
|
180
|
+
metricRows: Array<JSONObject>;
|
|
181
|
+
metricNameServiceNameMap: Dictionary<MetricType>;
|
|
182
|
+
}): Promise<void> {
|
|
183
|
+
if (
|
|
184
|
+
data.value === undefined ||
|
|
185
|
+
data.value === null ||
|
|
186
|
+
typeof data.value !== "number" ||
|
|
187
|
+
!isFinite(data.value)
|
|
188
|
+
) {
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const attributeInput: {
|
|
193
|
+
monitorId: ObjectID;
|
|
194
|
+
projectId: ObjectID;
|
|
195
|
+
monitorName?: string | undefined;
|
|
196
|
+
probeName?: string | undefined;
|
|
197
|
+
extraAttributes?: JSONObject;
|
|
198
|
+
} = {
|
|
199
|
+
monitorId: data.monitorId,
|
|
200
|
+
projectId: data.projectId,
|
|
201
|
+
monitorName: data.monitorName,
|
|
202
|
+
probeName: data.probeName,
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
if (data.extraAttributes) {
|
|
206
|
+
attributeInput.extraAttributes = data.extraAttributes;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const attributes: JSONObject =
|
|
210
|
+
this.buildMonitorMetricAttributes(attributeInput);
|
|
211
|
+
|
|
212
|
+
const metricRow: JSONObject = await this.buildMonitorMetricRow({
|
|
213
|
+
projectId: data.projectId,
|
|
214
|
+
monitorId: data.monitorId,
|
|
215
|
+
metricName: data.metricName,
|
|
216
|
+
value: data.value,
|
|
217
|
+
attributes: attributes,
|
|
218
|
+
metricPointType: data.metricPointType || MetricPointType.Sum,
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
data.metricRows.push(metricRow);
|
|
222
|
+
|
|
223
|
+
const metricType: MetricType = new MetricType();
|
|
224
|
+
metricType.name = data.metricName;
|
|
225
|
+
metricType.description = data.description;
|
|
226
|
+
metricType.unit = data.unit;
|
|
227
|
+
|
|
228
|
+
data.metricNameServiceNameMap[data.metricName] = metricType;
|
|
229
|
+
}
|
|
230
|
+
|
|
162
231
|
@CaptureSpan()
|
|
163
232
|
public static async saveMonitorMetrics(data: {
|
|
164
233
|
monitorId: ObjectID;
|
|
@@ -333,6 +402,359 @@ export default class MonitorMetricUtil {
|
|
|
333
402
|
metricNameServiceNameMap[MonitorMetricType.DiskUsagePercent] =
|
|
334
403
|
metricType;
|
|
335
404
|
}
|
|
405
|
+
|
|
406
|
+
// Per-disk I/O counters (cumulative since boot).
|
|
407
|
+
for (const diskMetric of basicMetrics.diskMetrics) {
|
|
408
|
+
const diskAttrs: JSONObject = {};
|
|
409
|
+
if (diskMetric.diskPath) {
|
|
410
|
+
diskAttrs["diskPath"] = diskMetric.diskPath;
|
|
411
|
+
}
|
|
412
|
+
if (diskMetric.device) {
|
|
413
|
+
diskAttrs["device"] = diskMetric.device;
|
|
414
|
+
}
|
|
415
|
+
if (diskMetric.fstype) {
|
|
416
|
+
diskAttrs["fstype"] = diskMetric.fstype;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
await this.pushMonitorMetric({
|
|
420
|
+
projectId: data.projectId,
|
|
421
|
+
monitorId: data.monitorId,
|
|
422
|
+
monitorName: data.monitorName,
|
|
423
|
+
probeName: data.probeName,
|
|
424
|
+
metricName: MonitorMetricType.DiskReadBytesTotal,
|
|
425
|
+
value: diskMetric.readBytes,
|
|
426
|
+
description: "Total bytes read from disk since boot",
|
|
427
|
+
unit: "bytes",
|
|
428
|
+
extraAttributes: diskAttrs,
|
|
429
|
+
metricRows,
|
|
430
|
+
metricNameServiceNameMap,
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
await this.pushMonitorMetric({
|
|
434
|
+
projectId: data.projectId,
|
|
435
|
+
monitorId: data.monitorId,
|
|
436
|
+
monitorName: data.monitorName,
|
|
437
|
+
probeName: data.probeName,
|
|
438
|
+
metricName: MonitorMetricType.DiskWriteBytesTotal,
|
|
439
|
+
value: diskMetric.writeBytes,
|
|
440
|
+
description: "Total bytes written to disk since boot",
|
|
441
|
+
unit: "bytes",
|
|
442
|
+
extraAttributes: diskAttrs,
|
|
443
|
+
metricRows,
|
|
444
|
+
metricNameServiceNameMap,
|
|
445
|
+
});
|
|
446
|
+
|
|
447
|
+
await this.pushMonitorMetric({
|
|
448
|
+
projectId: data.projectId,
|
|
449
|
+
monitorId: data.monitorId,
|
|
450
|
+
monitorName: data.monitorName,
|
|
451
|
+
probeName: data.probeName,
|
|
452
|
+
metricName: MonitorMetricType.DiskReadOpsTotal,
|
|
453
|
+
value: diskMetric.readCount,
|
|
454
|
+
description: "Total disk read operations since boot",
|
|
455
|
+
unit: "ops",
|
|
456
|
+
extraAttributes: diskAttrs,
|
|
457
|
+
metricRows,
|
|
458
|
+
metricNameServiceNameMap,
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
await this.pushMonitorMetric({
|
|
462
|
+
projectId: data.projectId,
|
|
463
|
+
monitorId: data.monitorId,
|
|
464
|
+
monitorName: data.monitorName,
|
|
465
|
+
probeName: data.probeName,
|
|
466
|
+
metricName: MonitorMetricType.DiskWriteOpsTotal,
|
|
467
|
+
value: diskMetric.writeCount,
|
|
468
|
+
description: "Total disk write operations since boot",
|
|
469
|
+
unit: "ops",
|
|
470
|
+
extraAttributes: diskAttrs,
|
|
471
|
+
metricRows,
|
|
472
|
+
metricNameServiceNameMap,
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
// Load average (1/5/15 min). Emitted only when the agent provides it.
|
|
478
|
+
if (basicMetrics.loadMetrics) {
|
|
479
|
+
await this.pushMonitorMetric({
|
|
480
|
+
projectId: data.projectId,
|
|
481
|
+
monitorId: data.monitorId,
|
|
482
|
+
monitorName: data.monitorName,
|
|
483
|
+
probeName: data.probeName,
|
|
484
|
+
metricName: MonitorMetricType.LoadAverage1Min,
|
|
485
|
+
value: basicMetrics.loadMetrics.load1,
|
|
486
|
+
description: "1-minute load average",
|
|
487
|
+
unit: "",
|
|
488
|
+
metricRows,
|
|
489
|
+
metricNameServiceNameMap,
|
|
490
|
+
});
|
|
491
|
+
await this.pushMonitorMetric({
|
|
492
|
+
projectId: data.projectId,
|
|
493
|
+
monitorId: data.monitorId,
|
|
494
|
+
monitorName: data.monitorName,
|
|
495
|
+
probeName: data.probeName,
|
|
496
|
+
metricName: MonitorMetricType.LoadAverage5Min,
|
|
497
|
+
value: basicMetrics.loadMetrics.load5,
|
|
498
|
+
description: "5-minute load average",
|
|
499
|
+
unit: "",
|
|
500
|
+
metricRows,
|
|
501
|
+
metricNameServiceNameMap,
|
|
502
|
+
});
|
|
503
|
+
await this.pushMonitorMetric({
|
|
504
|
+
projectId: data.projectId,
|
|
505
|
+
monitorId: data.monitorId,
|
|
506
|
+
monitorName: data.monitorName,
|
|
507
|
+
probeName: data.probeName,
|
|
508
|
+
metricName: MonitorMetricType.LoadAverage15Min,
|
|
509
|
+
value: basicMetrics.loadMetrics.load15,
|
|
510
|
+
description: "15-minute load average",
|
|
511
|
+
unit: "",
|
|
512
|
+
metricRows,
|
|
513
|
+
metricNameServiceNameMap,
|
|
514
|
+
});
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
// Memory extras: swap + available.
|
|
518
|
+
if (basicMetrics.memoryMetrics) {
|
|
519
|
+
await this.pushMonitorMetric({
|
|
520
|
+
projectId: data.projectId,
|
|
521
|
+
monitorId: data.monitorId,
|
|
522
|
+
monitorName: data.monitorName,
|
|
523
|
+
probeName: data.probeName,
|
|
524
|
+
metricName: MonitorMetricType.SwapUsagePercent,
|
|
525
|
+
value: basicMetrics.memoryMetrics.swapPercentUsed,
|
|
526
|
+
description: "Swap memory usage percentage",
|
|
527
|
+
unit: "%",
|
|
528
|
+
metricRows,
|
|
529
|
+
metricNameServiceNameMap,
|
|
530
|
+
});
|
|
531
|
+
await this.pushMonitorMetric({
|
|
532
|
+
projectId: data.projectId,
|
|
533
|
+
monitorId: data.monitorId,
|
|
534
|
+
monitorName: data.monitorName,
|
|
535
|
+
probeName: data.probeName,
|
|
536
|
+
metricName: MonitorMetricType.MemoryAvailableBytes,
|
|
537
|
+
value: basicMetrics.memoryMetrics.available,
|
|
538
|
+
description: "Memory available to new allocations",
|
|
539
|
+
unit: "bytes",
|
|
540
|
+
metricRows,
|
|
541
|
+
metricNameServiceNameMap,
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// CPU time breakdown — drills down into what the CPU was doing.
|
|
546
|
+
if (basicMetrics.cpuMetrics) {
|
|
547
|
+
await this.pushMonitorMetric({
|
|
548
|
+
projectId: data.projectId,
|
|
549
|
+
monitorId: data.monitorId,
|
|
550
|
+
monitorName: data.monitorName,
|
|
551
|
+
probeName: data.probeName,
|
|
552
|
+
metricName: MonitorMetricType.CPUTimeUserPercent,
|
|
553
|
+
value: basicMetrics.cpuMetrics.timeUserPercent,
|
|
554
|
+
description: "CPU time spent in user space",
|
|
555
|
+
unit: "%",
|
|
556
|
+
metricRows,
|
|
557
|
+
metricNameServiceNameMap,
|
|
558
|
+
});
|
|
559
|
+
await this.pushMonitorMetric({
|
|
560
|
+
projectId: data.projectId,
|
|
561
|
+
monitorId: data.monitorId,
|
|
562
|
+
monitorName: data.monitorName,
|
|
563
|
+
probeName: data.probeName,
|
|
564
|
+
metricName: MonitorMetricType.CPUTimeSystemPercent,
|
|
565
|
+
value: basicMetrics.cpuMetrics.timeSystemPercent,
|
|
566
|
+
description: "CPU time spent in kernel space",
|
|
567
|
+
unit: "%",
|
|
568
|
+
metricRows,
|
|
569
|
+
metricNameServiceNameMap,
|
|
570
|
+
});
|
|
571
|
+
await this.pushMonitorMetric({
|
|
572
|
+
projectId: data.projectId,
|
|
573
|
+
monitorId: data.monitorId,
|
|
574
|
+
monitorName: data.monitorName,
|
|
575
|
+
probeName: data.probeName,
|
|
576
|
+
metricName: MonitorMetricType.CPUTimeIoWaitPercent,
|
|
577
|
+
value: basicMetrics.cpuMetrics.timeIoWaitPercent,
|
|
578
|
+
description: "CPU time spent waiting on I/O",
|
|
579
|
+
unit: "%",
|
|
580
|
+
metricRows,
|
|
581
|
+
metricNameServiceNameMap,
|
|
582
|
+
});
|
|
583
|
+
await this.pushMonitorMetric({
|
|
584
|
+
projectId: data.projectId,
|
|
585
|
+
monitorId: data.monitorId,
|
|
586
|
+
monitorName: data.monitorName,
|
|
587
|
+
probeName: data.probeName,
|
|
588
|
+
metricName: MonitorMetricType.CPUTimeIdlePercent,
|
|
589
|
+
value: basicMetrics.cpuMetrics.timeIdlePercent,
|
|
590
|
+
description: "CPU time spent idle",
|
|
591
|
+
unit: "%",
|
|
592
|
+
metricRows,
|
|
593
|
+
metricNameServiceNameMap,
|
|
594
|
+
});
|
|
595
|
+
await this.pushMonitorMetric({
|
|
596
|
+
projectId: data.projectId,
|
|
597
|
+
monitorId: data.monitorId,
|
|
598
|
+
monitorName: data.monitorName,
|
|
599
|
+
probeName: data.probeName,
|
|
600
|
+
metricName: MonitorMetricType.CPUTimeStealPercent,
|
|
601
|
+
value: basicMetrics.cpuMetrics.timeStealPercent,
|
|
602
|
+
description: "CPU time stolen by the hypervisor",
|
|
603
|
+
unit: "%",
|
|
604
|
+
metricRows,
|
|
605
|
+
metricNameServiceNameMap,
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
// Network counters — both per-interface and aggregate.
|
|
610
|
+
if (basicMetrics.networkMetrics) {
|
|
611
|
+
const net: typeof basicMetrics.networkMetrics =
|
|
612
|
+
basicMetrics.networkMetrics;
|
|
613
|
+
|
|
614
|
+
for (const iface of net.interfaces || []) {
|
|
615
|
+
const ifaceAttrs: JSONObject = {
|
|
616
|
+
interfaceName: (iface as NetworkInterfaceMetrics).interfaceName,
|
|
617
|
+
};
|
|
618
|
+
|
|
619
|
+
await this.pushMonitorMetric({
|
|
620
|
+
projectId: data.projectId,
|
|
621
|
+
monitorId: data.monitorId,
|
|
622
|
+
monitorName: data.monitorName,
|
|
623
|
+
probeName: data.probeName,
|
|
624
|
+
metricName: MonitorMetricType.NetworkBytesReceivedTotal,
|
|
625
|
+
value: iface.bytesReceived,
|
|
626
|
+
description: "Network bytes received since boot (per-interface)",
|
|
627
|
+
unit: "bytes",
|
|
628
|
+
extraAttributes: ifaceAttrs,
|
|
629
|
+
metricRows,
|
|
630
|
+
metricNameServiceNameMap,
|
|
631
|
+
});
|
|
632
|
+
await this.pushMonitorMetric({
|
|
633
|
+
projectId: data.projectId,
|
|
634
|
+
monitorId: data.monitorId,
|
|
635
|
+
monitorName: data.monitorName,
|
|
636
|
+
probeName: data.probeName,
|
|
637
|
+
metricName: MonitorMetricType.NetworkBytesSentTotal,
|
|
638
|
+
value: iface.bytesSent,
|
|
639
|
+
description: "Network bytes sent since boot (per-interface)",
|
|
640
|
+
unit: "bytes",
|
|
641
|
+
extraAttributes: ifaceAttrs,
|
|
642
|
+
metricRows,
|
|
643
|
+
metricNameServiceNameMap,
|
|
644
|
+
});
|
|
645
|
+
await this.pushMonitorMetric({
|
|
646
|
+
projectId: data.projectId,
|
|
647
|
+
monitorId: data.monitorId,
|
|
648
|
+
monitorName: data.monitorName,
|
|
649
|
+
probeName: data.probeName,
|
|
650
|
+
metricName: MonitorMetricType.NetworkPacketsReceivedTotal,
|
|
651
|
+
value: iface.packetsReceived,
|
|
652
|
+
description: "Network packets received since boot (per-interface)",
|
|
653
|
+
unit: "packets",
|
|
654
|
+
extraAttributes: ifaceAttrs,
|
|
655
|
+
metricRows,
|
|
656
|
+
metricNameServiceNameMap,
|
|
657
|
+
});
|
|
658
|
+
await this.pushMonitorMetric({
|
|
659
|
+
projectId: data.projectId,
|
|
660
|
+
monitorId: data.monitorId,
|
|
661
|
+
monitorName: data.monitorName,
|
|
662
|
+
probeName: data.probeName,
|
|
663
|
+
metricName: MonitorMetricType.NetworkPacketsSentTotal,
|
|
664
|
+
value: iface.packetsSent,
|
|
665
|
+
description: "Network packets sent since boot (per-interface)",
|
|
666
|
+
unit: "packets",
|
|
667
|
+
extraAttributes: ifaceAttrs,
|
|
668
|
+
metricRows,
|
|
669
|
+
metricNameServiceNameMap,
|
|
670
|
+
});
|
|
671
|
+
await this.pushMonitorMetric({
|
|
672
|
+
projectId: data.projectId,
|
|
673
|
+
monitorId: data.monitorId,
|
|
674
|
+
monitorName: data.monitorName,
|
|
675
|
+
probeName: data.probeName,
|
|
676
|
+
metricName: MonitorMetricType.NetworkErrorsIn,
|
|
677
|
+
value: iface.errorsIn,
|
|
678
|
+
description: "Network receive errors since boot (per-interface)",
|
|
679
|
+
unit: "errors",
|
|
680
|
+
extraAttributes: ifaceAttrs,
|
|
681
|
+
metricRows,
|
|
682
|
+
metricNameServiceNameMap,
|
|
683
|
+
});
|
|
684
|
+
await this.pushMonitorMetric({
|
|
685
|
+
projectId: data.projectId,
|
|
686
|
+
monitorId: data.monitorId,
|
|
687
|
+
monitorName: data.monitorName,
|
|
688
|
+
probeName: data.probeName,
|
|
689
|
+
metricName: MonitorMetricType.NetworkErrorsOut,
|
|
690
|
+
value: iface.errorsOut,
|
|
691
|
+
description: "Network transmit errors since boot (per-interface)",
|
|
692
|
+
unit: "errors",
|
|
693
|
+
extraAttributes: ifaceAttrs,
|
|
694
|
+
metricRows,
|
|
695
|
+
metricNameServiceNameMap,
|
|
696
|
+
});
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
await this.pushMonitorMetric({
|
|
700
|
+
projectId: data.projectId,
|
|
701
|
+
monitorId: data.monitorId,
|
|
702
|
+
monitorName: data.monitorName,
|
|
703
|
+
probeName: data.probeName,
|
|
704
|
+
metricName: MonitorMetricType.NetworkConnectionsEstablished,
|
|
705
|
+
value: net.connectionsEstablished,
|
|
706
|
+
description: "Count of ESTABLISHED network connections",
|
|
707
|
+
unit: "connections",
|
|
708
|
+
metricRows,
|
|
709
|
+
metricNameServiceNameMap,
|
|
710
|
+
});
|
|
711
|
+
await this.pushMonitorMetric({
|
|
712
|
+
projectId: data.projectId,
|
|
713
|
+
monitorId: data.monitorId,
|
|
714
|
+
monitorName: data.monitorName,
|
|
715
|
+
probeName: data.probeName,
|
|
716
|
+
metricName: MonitorMetricType.NetworkConnectionsListen,
|
|
717
|
+
value: net.connectionsListen,
|
|
718
|
+
description: "Count of LISTENing network sockets",
|
|
719
|
+
unit: "connections",
|
|
720
|
+
metricRows,
|
|
721
|
+
metricNameServiceNameMap,
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
// Host uptime.
|
|
726
|
+
if (basicMetrics.hostMetrics) {
|
|
727
|
+
await this.pushMonitorMetric({
|
|
728
|
+
projectId: data.projectId,
|
|
729
|
+
monitorId: data.monitorId,
|
|
730
|
+
monitorName: data.monitorName,
|
|
731
|
+
probeName: data.probeName,
|
|
732
|
+
metricName: MonitorMetricType.HostUptimeSeconds,
|
|
733
|
+
value: basicMetrics.hostMetrics.uptimeSeconds,
|
|
734
|
+
description: "Host uptime in seconds",
|
|
735
|
+
unit: "seconds",
|
|
736
|
+
metricRows,
|
|
737
|
+
metricNameServiceNameMap,
|
|
738
|
+
});
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// Process count — simple scalar derived from the processes array.
|
|
742
|
+
const serverProcesses: Array<unknown> | undefined = (
|
|
743
|
+
data.dataToProcess as ServerMonitorResponse
|
|
744
|
+
).processes;
|
|
745
|
+
if (serverProcesses && serverProcesses.length >= 0) {
|
|
746
|
+
await this.pushMonitorMetric({
|
|
747
|
+
projectId: data.projectId,
|
|
748
|
+
monitorId: data.monitorId,
|
|
749
|
+
monitorName: data.monitorName,
|
|
750
|
+
probeName: data.probeName,
|
|
751
|
+
metricName: MonitorMetricType.ProcessCountTotal,
|
|
752
|
+
value: serverProcesses.length,
|
|
753
|
+
description: "Total running processes on the server",
|
|
754
|
+
unit: "processes",
|
|
755
|
+
metricRows,
|
|
756
|
+
metricNameServiceNameMap,
|
|
757
|
+
});
|
|
336
758
|
}
|
|
337
759
|
}
|
|
338
760
|
|
|
@@ -709,6 +709,7 @@ export default class MonitorResourceUtil {
|
|
|
709
709
|
props: {
|
|
710
710
|
telemetryQuery: telemetryQuery,
|
|
711
711
|
},
|
|
712
|
+
matchesPerSeries: response.perSeriesMatches,
|
|
712
713
|
});
|
|
713
714
|
|
|
714
715
|
await MonitorAlert.criteriaMetCreateAlertsAndUpdateMonitorStatus({
|
|
@@ -721,6 +722,7 @@ export default class MonitorResourceUtil {
|
|
|
721
722
|
props: {
|
|
722
723
|
telemetryQuery: telemetryQuery,
|
|
723
724
|
},
|
|
725
|
+
matchesPerSeries: response.perSeriesMatches,
|
|
724
726
|
});
|
|
725
727
|
} else if (
|
|
726
728
|
!response.criteriaMetId &&
|