@leverege/build-tools 2.66.0 → 2.66.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh +1 -1
- package/src/helm-charts/prom-operator/rules/awesome-base/elasticsearch-rules.yaml +219 -0
- package/src/helm-charts/prom-operator/rules/awesome-base/etcd-exporter.yaml +153 -0
- package/src/helm-charts/prom-operator/rules/awesome-base/google-cadvisor.yaml +108 -0
- package/src/helm-charts/prom-operator/rules/{new-awesome.yaml → awesome-base/kubestate-exporter.yaml} +2 -1
- package/src/helm-charts/prom-operator/rules/awesome-base/prometheus-rules.yaml +318 -0
- package/src/helm-charts/prom-operator/rules/awesome-base/redis-rules.yaml +142 -0
- package/src/helm-charts/prom-operator/rules/awesome-base/traefik-rules.yaml +43 -0
- package/src/helm-charts/prom-operator/rules/cadvisor-rules.yaml +109 -0
- package/src/helm-charts/prom-operator/rules/elasticsearch-rules.yaml +213 -95
- package/src/helm-charts/prom-operator/rules/etcd-rulte.yaml +153 -0
- package/src/helm-charts/prom-operator/rules/kubestate-rules.yaml +412 -0
- package/src/helm-charts/prom-operator/rules/prometheus-rules.yaml +312 -73
- package/src/helm-charts/prom-operator/rules/redis-rules.yaml +136 -75
- package/src/helm-charts/prom-operator/rules/traefik-rules.yaml +48 -21
- package/src/helmup.sh +1 -1
- package/src/helm-charts/prom-operator/rules/awesome-k8s.yaml +0 -326
- package/src/helm-charts/prom-operator/rules/kubernetes-rules.yaml +0 -109
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
# https://samber.github.io/awesome-prometheus-alerts/rules#kubernetes
|
|
2
|
+
apiVersion: monitoring.coreos.com/v1
|
|
3
|
+
kind: PrometheusRule
|
|
4
|
+
metadata:
|
|
5
|
+
name: kubestate-exporter
|
|
6
|
+
namespace: prometheus
|
|
7
|
+
spec:
|
|
8
|
+
groups:
|
|
9
|
+
- name: Kubestate (awesome) # KubestateExporter-rules
|
|
10
|
+
rules:
|
|
11
|
+
- alert: KubernetesNodeNotReady
|
|
12
|
+
expr: kube_node_status_condition{condition="Ready",status="true"} == 0
|
|
13
|
+
for: 10m
|
|
14
|
+
labels:
|
|
15
|
+
severity: critical
|
|
16
|
+
annotations:
|
|
17
|
+
summary: Kubernetes Node ready (node {{ $labels.node }})
|
|
18
|
+
description: |-
|
|
19
|
+
Node {{ $labels.node }} has been unready for a long time
|
|
20
|
+
VALUE = {{ $value }}
|
|
21
|
+
LABELS = {{ $labels }}
|
|
22
|
+
- alert: KubernetesNodeMemoryPressure
|
|
23
|
+
expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
|
|
24
|
+
for: 2m
|
|
25
|
+
labels:
|
|
26
|
+
severity: critical
|
|
27
|
+
annotations:
|
|
28
|
+
summary: Kubernetes memory pressure (node {{ $labels.node }})
|
|
29
|
+
description: |-
|
|
30
|
+
Node {{ $labels.node }} has MemoryPressure condition
|
|
31
|
+
VALUE = {{ $value }}
|
|
32
|
+
LABELS = {{ $labels }}
|
|
33
|
+
- alert: KubernetesNodeDiskPressure
|
|
34
|
+
expr: kube_node_status_condition{condition="DiskPressure",status="true",node!~".*preemptible.*"} == 1 # awesome mod => added preemptible check
|
|
35
|
+
for: 2m
|
|
36
|
+
labels:
|
|
37
|
+
severity: critical
|
|
38
|
+
annotations:
|
|
39
|
+
summary: Kubernetes disk pressure (node {{ $labels.node }})
|
|
40
|
+
description: |-
|
|
41
|
+
Node {{ $labels.node }} has DiskPressure condition
|
|
42
|
+
VALUE = {{ $value }}
|
|
43
|
+
LABELS = {{ $labels }}
|
|
44
|
+
- alert: KubernetesNodeNetworkUnavailable
|
|
45
|
+
expr: kube_node_status_condition{condition="NetworkUnavailable",status="true"} == 1
|
|
46
|
+
for: 2m
|
|
47
|
+
labels:
|
|
48
|
+
severity: critical
|
|
49
|
+
annotations:
|
|
50
|
+
summary: Kubernetes Node network unavailable (instance {{ $labels.instance }})
|
|
51
|
+
description: |-
|
|
52
|
+
Node {{ $labels.node }} has NetworkUnavailable condition
|
|
53
|
+
VALUE = {{ $value }}
|
|
54
|
+
LABELS = {{ $labels }}
|
|
55
|
+
- alert: KubernetesNodeOutOfPodCapacity
|
|
56
|
+
expr: sum by (node) ((kube_pod_status_phase{phase="Running"} == 1) + on(uid, instance) group_left(node) (0 * kube_pod_info{pod_template_hash=""})) / sum by (node) (kube_node_status_allocatable{resource="pods"}) * 100 > 90
|
|
57
|
+
for: 2m
|
|
58
|
+
labels:
|
|
59
|
+
severity: warning
|
|
60
|
+
annotations:
|
|
61
|
+
summary: Kubernetes Node out of pod capacity (instance {{ $labels.instance }})
|
|
62
|
+
description: |-
|
|
63
|
+
Node {{ $labels.node }} is out of pod capacity
|
|
64
|
+
VALUE = {{ $value }}
|
|
65
|
+
LABELS = {{ $labels }}
|
|
66
|
+
- alert: KubernetesContainerOomKiller
|
|
67
|
+
expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1
|
|
68
|
+
for: 0m
|
|
69
|
+
labels:
|
|
70
|
+
severity: warning
|
|
71
|
+
annotations:
|
|
72
|
+
summary: Kubernetes container oom killer ({{ $labels.namespace }}/{{ $labels.pod }}:{{ $labels.container }})
|
|
73
|
+
description: |-
|
|
74
|
+
Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.
|
|
75
|
+
VALUE = {{ $value }}
|
|
76
|
+
LABELS = {{ $labels }}
|
|
77
|
+
- alert: KubernetesJobFailed
|
|
78
|
+
expr: kube_job_status_failed > 0
|
|
79
|
+
for: 0m
|
|
80
|
+
labels:
|
|
81
|
+
severity: warning
|
|
82
|
+
annotations:
|
|
83
|
+
summary: Kubernetes Job failed ({{ $labels.namespace }}/{{ $labels.job_name }})
|
|
84
|
+
description: |-
|
|
85
|
+
Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete
|
|
86
|
+
VALUE = {{ $value }}
|
|
87
|
+
LABELS = {{ $labels }}
|
|
88
|
+
- alert: KubernetesJobNotStarting
|
|
89
|
+
expr: kube_job_status_active == 0 and kube_job_status_failed == 0 and kube_job_status_succeeded == 0 and (time() - kube_job_status_start_time) > 600
|
|
90
|
+
for: 0m
|
|
91
|
+
labels:
|
|
92
|
+
severity: warning
|
|
93
|
+
annotations:
|
|
94
|
+
summary: Kubernetes Job not starting ({{ $labels.namespace }}/{{ $labels.job_name }})
|
|
95
|
+
description: |-
|
|
96
|
+
Job {{ $labels.namespace }}/{{ $labels.job_name }} did not start for 10 minutes
|
|
97
|
+
VALUE = {{ $value }}
|
|
98
|
+
LABELS = {{ $labels }}
|
|
99
|
+
- alert: KubernetesCronjobSuspended
|
|
100
|
+
expr: kube_cronjob_spec_suspend != 0
|
|
101
|
+
for: 0m
|
|
102
|
+
labels:
|
|
103
|
+
severity: warning
|
|
104
|
+
annotations:
|
|
105
|
+
summary: Kubernetes CronJob suspended ({{ $labels.namespace }}/{{ $labels.cronjob }})
|
|
106
|
+
description: |-
|
|
107
|
+
CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is suspended
|
|
108
|
+
VALUE = {{ $value }}
|
|
109
|
+
LABELS = {{ $labels }}
|
|
110
|
+
- alert: KubernetesPersistentvolumeclaimPending
|
|
111
|
+
expr: kube_persistentvolumeclaim_status_phase{phase="Pending"} == 1
|
|
112
|
+
for: 2m
|
|
113
|
+
labels:
|
|
114
|
+
severity: warning
|
|
115
|
+
annotations:
|
|
116
|
+
summary: Kubernetes PersistentVolumeClaim pending ({{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }})
|
|
117
|
+
description: |-
|
|
118
|
+
PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending
|
|
119
|
+
VALUE = {{ $value }}
|
|
120
|
+
LABELS = {{ $labels }}
|
|
121
|
+
- alert: KubernetesVolumeOutOfDiskSpace
|
|
122
|
+
expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10
|
|
123
|
+
for: 2m
|
|
124
|
+
labels:
|
|
125
|
+
severity: warning
|
|
126
|
+
annotations:
|
|
127
|
+
summary: Kubernetes Volume out of disk space (instance {{ $labels.instance }})
|
|
128
|
+
description: |-
|
|
129
|
+
Volume is almost full (< 10% left)
|
|
130
|
+
VALUE = {{ $value }}
|
|
131
|
+
LABELS = {{ $labels }}
|
|
132
|
+
# Replaced by custom rule - see below
|
|
133
|
+
# - alert: KubernetesVolumeFullInFourDays
|
|
134
|
+
# expr: predict_linear(kubelet_volume_stats_available_bytes[6h:5m], 4 * 24 * 3600) < 0
|
|
135
|
+
# for: 0m
|
|
136
|
+
# labels:
|
|
137
|
+
# severity: critical
|
|
138
|
+
# annotations:
|
|
139
|
+
# summary: Kubernetes Volume full in four days (instance {{ $labels.instance }})
|
|
140
|
+
# description: |-
|
|
141
|
+
# Volume under {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available.
|
|
142
|
+
# VALUE = {{ $value }}
|
|
143
|
+
# LABELS = {{ $labels }}
|
|
144
|
+
- alert: KubernetesPersistentvolumeError
|
|
145
|
+
expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending", job="kube-state-metrics"} > 0
|
|
146
|
+
for: 0m
|
|
147
|
+
labels:
|
|
148
|
+
severity: critical
|
|
149
|
+
annotations:
|
|
150
|
+
summary: Kubernetes PersistentVolumeClaim pending ({{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }})
|
|
151
|
+
description: |-
|
|
152
|
+
Persistent volume {{ $labels.persistentvolume }} is in bad state
|
|
153
|
+
VALUE = {{ $value }}
|
|
154
|
+
LABELS = {{ $labels }}
|
|
155
|
+
- alert: KubernetesStatefulsetDown
|
|
156
|
+
expr: kube_statefulset_replicas != kube_statefulset_status_replicas_ready > 0
|
|
157
|
+
for: 1m
|
|
158
|
+
labels:
|
|
159
|
+
severity: critical
|
|
160
|
+
annotations:
|
|
161
|
+
summary: Kubernetes StatefulSet down ({{ $labels.namespace }}/{{ $labels.statefulset }})
|
|
162
|
+
description: |-
|
|
163
|
+
StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} went down
|
|
164
|
+
VALUE = {{ $value }}
|
|
165
|
+
LABELS = {{ $labels }}
|
|
166
|
+
- alert: KubernetesHpaScaleInability
|
|
167
|
+
expr: (kube_horizontalpodautoscaler_spec_max_replicas - kube_horizontalpodautoscaler_status_desired_replicas) * on (horizontalpodautoscaler,namespace) (kube_horizontalpodautoscaler_status_condition{condition="ScalingLimited", status="true"} == 1) == 0
|
|
168
|
+
for: 2m
|
|
169
|
+
labels:
|
|
170
|
+
severity: warning
|
|
171
|
+
annotations:
|
|
172
|
+
summary: Kubernetes HPA scale inability (instance {{ $labels.instance }})
|
|
173
|
+
description: |-
|
|
174
|
+
HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to scale
|
|
175
|
+
VALUE = {{ $value }}
|
|
176
|
+
LABELS = {{ $labels }}
|
|
177
|
+
- alert: KubernetesHpaMetricsUnavailability
|
|
178
|
+
expr: kube_horizontalpodautoscaler_status_condition{status="false", condition="ScalingActive"} == 1
|
|
179
|
+
for: 0m
|
|
180
|
+
labels:
|
|
181
|
+
severity: warning
|
|
182
|
+
annotations:
|
|
183
|
+
summary: Kubernetes HPA metrics unavailability (instance {{ $labels.instance }})
|
|
184
|
+
description: |-
|
|
185
|
+
HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to collect metrics
|
|
186
|
+
VALUE = {{ $value }}
|
|
187
|
+
LABELS = {{ $labels }}
|
|
188
|
+
- alert: KubernetesHpaScaleMaximum
|
|
189
|
+
expr: (kube_horizontalpodautoscaler_status_desired_replicas >= kube_horizontalpodautoscaler_spec_max_replicas) and (kube_horizontalpodautoscaler_spec_max_replicas > 1) and (kube_horizontalpodautoscaler_spec_min_replicas != kube_horizontalpodautoscaler_spec_max_replicas)
|
|
190
|
+
for: 2m
|
|
191
|
+
labels:
|
|
192
|
+
severity: info
|
|
193
|
+
annotations:
|
|
194
|
+
summary: Kubernetes HPA scale maximum (instance {{ $labels.instance }})
|
|
195
|
+
description: |-
|
|
196
|
+
HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has hit maximum number of desired pods
|
|
197
|
+
VALUE = {{ $value }}
|
|
198
|
+
LABELS = {{ $labels }}
|
|
199
|
+
- alert: KubernetesHpaUnderutilized
|
|
200
|
+
expr: max(quantile_over_time(0.5, kube_horizontalpodautoscaler_status_desired_replicas[1d]) == kube_horizontalpodautoscaler_spec_min_replicas) by (horizontalpodautoscaler) > 3
|
|
201
|
+
for: 0m
|
|
202
|
+
labels:
|
|
203
|
+
severity: info
|
|
204
|
+
annotations:
|
|
205
|
+
summary: Kubernetes HPA underutilized (instance {{ $labels.instance }})
|
|
206
|
+
description: |-
|
|
207
|
+
HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is constantly at minimum replicas for 50% of the time. Potential cost saving here.
|
|
208
|
+
VALUE = {{ $value }}
|
|
209
|
+
LABELS = {{ $labels }}
|
|
210
|
+
- alert: KubernetesPodNotHealthy
|
|
211
|
+
expr: sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"}) > 0
|
|
212
|
+
for: 15m
|
|
213
|
+
labels:
|
|
214
|
+
severity: critical
|
|
215
|
+
annotations:
|
|
216
|
+
summary: Kubernetes Pod not healthy ({{ $labels.namespace }}/{{ $labels.pod }})
|
|
217
|
+
description: |-
|
|
218
|
+
Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-running state for longer than 15 minutes.
|
|
219
|
+
VALUE = {{ $value }}
|
|
220
|
+
LABELS = {{ $labels }}
|
|
221
|
+
- alert: KubernetesPodCrashLooping
|
|
222
|
+
expr: increase(kube_pod_container_status_restarts_total[1m]) > 3
|
|
223
|
+
for: 2m
|
|
224
|
+
labels:
|
|
225
|
+
severity: warning
|
|
226
|
+
annotations:
|
|
227
|
+
summary: Kubernetes pod crash looping ({{ $labels.namespace }}/{{ $labels.pod }})
|
|
228
|
+
description: |-
|
|
229
|
+
Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping
|
|
230
|
+
VALUE = {{ $value }}
|
|
231
|
+
LABELS = {{ $labels }}
|
|
232
|
+
- alert: KubernetesReplicasetReplicasMismatch
|
|
233
|
+
expr: kube_replicaset_spec_replicas != kube_replicaset_status_ready_replicas
|
|
234
|
+
for: 10m
|
|
235
|
+
labels:
|
|
236
|
+
severity: warning
|
|
237
|
+
annotations:
|
|
238
|
+
summary: Kubernetes ReplicasSet mismatch ({{ $labels.namespace }}/{{ $labels.replicaset }})
|
|
239
|
+
description: |-
|
|
240
|
+
ReplicaSet {{ $labels.namespace }}/{{ $labels.replicaset }} replicas mismatch
|
|
241
|
+
VALUE = {{ $value }}
|
|
242
|
+
LABELS = {{ $labels }}
|
|
243
|
+
- alert: KubernetesDeploymentReplicasMismatch
|
|
244
|
+
expr: kube_deployment_spec_replicas != kube_deployment_status_replicas_available
|
|
245
|
+
for: 10m
|
|
246
|
+
labels:
|
|
247
|
+
severity: warning
|
|
248
|
+
annotations:
|
|
249
|
+
summary: Kubernetes Deployment replicas mismatch ({{ $labels.namespace }}/{{ $labels.deployment }})
|
|
250
|
+
description: |-
|
|
251
|
+
Deployment {{ $labels.namespace }}/{{ $labels.deployment }} replicas mismatch
|
|
252
|
+
VALUE = {{ $value }}
|
|
253
|
+
LABELS = {{ $labels }}
|
|
254
|
+
- alert: KubernetesStatefulsetReplicasMismatch
|
|
255
|
+
expr: kube_statefulset_status_replicas_ready != kube_statefulset_status_replicas
|
|
256
|
+
for: 10m
|
|
257
|
+
labels:
|
|
258
|
+
severity: warning
|
|
259
|
+
annotations:
|
|
260
|
+
summary: Kubernetes StatefulSet replicas mismatch (instance {{ $labels.instance }})
|
|
261
|
+
description: |-
|
|
262
|
+
StatefulSet does not match the expected number of replicas.
|
|
263
|
+
VALUE = {{ $value }}
|
|
264
|
+
LABELS = {{ $labels }}
|
|
265
|
+
- alert: KubernetesDeploymentGenerationMismatch
|
|
266
|
+
expr: kube_deployment_status_observed_generation != kube_deployment_metadata_generation
|
|
267
|
+
for: 10m
|
|
268
|
+
labels:
|
|
269
|
+
severity: critical
|
|
270
|
+
annotations:
|
|
271
|
+
summary: Kubernetes Deployment generation mismatch ({{ $labels.namespace }}/{{ $labels.deployment }})
|
|
272
|
+
description: |-
|
|
273
|
+
Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has failed but has not been rolled back.
|
|
274
|
+
VALUE = {{ $value }}
|
|
275
|
+
LABELS = {{ $labels }}
|
|
276
|
+
- alert: KubernetesStatefulsetGenerationMismatch
|
|
277
|
+
expr: kube_statefulset_status_observed_generation != kube_statefulset_metadata_generation
|
|
278
|
+
for: 10m
|
|
279
|
+
labels:
|
|
280
|
+
severity: critical
|
|
281
|
+
annotations:
|
|
282
|
+
summary: Kubernetes StatefulSet generation mismatch ({{ $labels.namespace }}/{{ $labels.statefulset }})
|
|
283
|
+
description: |-
|
|
284
|
+
StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has failed but has not been rolled back.
|
|
285
|
+
VALUE = {{ $value }}
|
|
286
|
+
LABELS = {{ $labels }}
|
|
287
|
+
- alert: KubernetesStatefulsetUpdateNotRolledOut
|
|
288
|
+
expr: max without (revision) (kube_statefulset_status_current_revision unless kube_statefulset_status_update_revision) * (kube_statefulset_replicas != kube_statefulset_status_replicas_updated)
|
|
289
|
+
for: 10m
|
|
290
|
+
labels:
|
|
291
|
+
severity: warning
|
|
292
|
+
annotations:
|
|
293
|
+
summary: Kubernetes StatefulSet update not rolled out ({{ $labels.namespace }}/{{ $labels.statefulset }})
|
|
294
|
+
description: |-
|
|
295
|
+
StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.
|
|
296
|
+
VALUE = {{ $value }}
|
|
297
|
+
LABELS = {{ $labels }}
|
|
298
|
+
- alert: KubernetesDaemonsetRolloutStuck
|
|
299
|
+
expr: kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100 or kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0
|
|
300
|
+
for: 10m
|
|
301
|
+
labels:
|
|
302
|
+
severity: warning
|
|
303
|
+
annotations:
|
|
304
|
+
summary: Kubernetes DaemonSet rollout stuck ({{ $labels.namespace }}/{{ $labels.daemonset }})
|
|
305
|
+
description: |-
|
|
306
|
+
Some Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled or not ready
|
|
307
|
+
VALUE = {{ $value }}
|
|
308
|
+
LABELS = {{ $labels }}
|
|
309
|
+
- alert: KubernetesDaemonsetMisscheduled
|
|
310
|
+
expr: kube_daemonset_status_number_misscheduled > 0
|
|
311
|
+
for: 1m
|
|
312
|
+
labels:
|
|
313
|
+
severity: critical
|
|
314
|
+
annotations:
|
|
315
|
+
summary: Kubernetes DaemonSet misscheduled ({{ $labels.namespace }}/{{ $labels.daemonset }})
|
|
316
|
+
description: |-
|
|
317
|
+
Some Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run
|
|
318
|
+
VALUE = {{ $value }}
|
|
319
|
+
LABELS = {{ $labels }}
|
|
320
|
+
- alert: KubernetesCronjobTooLong
|
|
321
|
+
expr: time() - kube_cronjob_next_schedule_time > 3600
|
|
322
|
+
for: 0m
|
|
323
|
+
labels:
|
|
324
|
+
severity: warning
|
|
325
|
+
annotations:
|
|
326
|
+
summary: Kubernetes CronJob too long ({{ $labels.namespace }}/{{ $labels.cronjob }})
|
|
327
|
+
description: |-
|
|
328
|
+
CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete.
|
|
329
|
+
VALUE = {{ $value }}
|
|
330
|
+
LABELS = {{ $labels }}
|
|
331
|
+
- alert: KubernetesJobSlowCompletion
|
|
332
|
+
expr: kube_job_spec_completions - kube_job_status_succeeded - kube_job_status_failed > 0
|
|
333
|
+
for: 12h
|
|
334
|
+
labels:
|
|
335
|
+
severity: critical
|
|
336
|
+
annotations:
|
|
337
|
+
summary: Kubernetes job slow completion ({{ $labels.namespace }}/{{ $labels.job_name }})
|
|
338
|
+
description: |-
|
|
339
|
+
Kubernetes Job {{ $labels.namespace }}/{{ $labels.job_name }} did not complete in time.
|
|
340
|
+
VALUE = {{ $value }}
|
|
341
|
+
LABELS = {{ $labels }}
|
|
342
|
+
- alert: KubernetesApiServerErrors
|
|
343
|
+
expr: sum(rate(apiserver_request_total{job="apiserver",code=~"(?:5..)"}[1m])) by (instance, job) / sum(rate(apiserver_request_total{job="apiserver"}[1m])) by (instance, job) * 100 > 3
|
|
344
|
+
for: 2m
|
|
345
|
+
labels:
|
|
346
|
+
severity: critical
|
|
347
|
+
annotations:
|
|
348
|
+
summary: Kubernetes API server errors (instance {{ $labels.instance }})
|
|
349
|
+
description: |-
|
|
350
|
+
Kubernetes API server is experiencing high error rate
|
|
351
|
+
VALUE = {{ $value }}
|
|
352
|
+
LABELS = {{ $labels }}
|
|
353
|
+
- alert: KubernetesApiClientErrors
|
|
354
|
+
expr: (sum(rate(rest_client_requests_total{code=~"(4|5).."}[1m])) by (instance, job) / sum(rate(rest_client_requests_total[1m])) by (instance, job)) * 100 > 1
|
|
355
|
+
for: 5m
|
|
356
|
+
labels:
|
|
357
|
+
severity: critical
|
|
358
|
+
annotations:
|
|
359
|
+
summary: Kubernetes API client errors (instance {{ $labels.instance }})
|
|
360
|
+
description: |-
|
|
361
|
+
Kubernetes API client is experiencing high error rate
|
|
362
|
+
VALUE = {{ $value }}
|
|
363
|
+
LABELS = {{ $labels }}
|
|
364
|
+
- alert: KubernetesClientCertificateExpiresNextWeek
|
|
365
|
+
expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 7*24*60*60
|
|
366
|
+
for: 0m
|
|
367
|
+
labels:
|
|
368
|
+
severity: warning
|
|
369
|
+
annotations:
|
|
370
|
+
summary: Kubernetes client certificate expires next week (instance {{ $labels.instance }})
|
|
371
|
+
description: |-
|
|
372
|
+
A client certificate used to authenticate to the apiserver is expiring next week.
|
|
373
|
+
VALUE = {{ $value }}
|
|
374
|
+
LABELS = {{ $labels }}
|
|
375
|
+
- alert: KubernetesClientCertificateExpiresSoon
|
|
376
|
+
expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 24*60*60
|
|
377
|
+
for: 0m
|
|
378
|
+
labels:
|
|
379
|
+
severity: critical
|
|
380
|
+
annotations:
|
|
381
|
+
summary: Kubernetes client certificate expires soon (instance {{ $labels.instance }})
|
|
382
|
+
description: |-
|
|
383
|
+
A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.
|
|
384
|
+
VALUE = {{ $value }}
|
|
385
|
+
LABELS = {{ $labels }}
|
|
386
|
+
- alert: KubernetesApiServerLatency
|
|
387
|
+
expr: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!~"(?:CONNECT|WATCHLIST|WATCH|PROXY)"} [10m])) WITHOUT (subresource)) > 1
|
|
388
|
+
for: 2m
|
|
389
|
+
labels:
|
|
390
|
+
severity: warning
|
|
391
|
+
annotations:
|
|
392
|
+
summary: Kubernetes API server latency (instance {{ $labels.instance }})
|
|
393
|
+
description: |-
|
|
394
|
+
Kubernetes API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.
|
|
395
|
+
VALUE = {{ $value }}
|
|
396
|
+
LABELS = {{ $labels }}
|
|
397
|
+
#
|
|
398
|
+
# Custom or "tuned" alerts
|
|
399
|
+
#
|
|
400
|
+
- alert: KubernetesVolumeUsageHigh
|
|
401
|
+
expr: (1 - kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes) > 0.85
|
|
402
|
+
for: 30m
|
|
403
|
+
labels:
|
|
404
|
+
severity: warning
|
|
405
|
+
annotations:
|
|
406
|
+
summary: PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is over 85% full
|
|
407
|
+
description: |
|
|
408
|
+
Persistent volume claim {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} is over 85% used and has less than 20GiB of free space.
|
|
409
|
+
This may lead to application failure if not addressed.
|
|
410
|
+
|
|
411
|
+
VALUE = {{ $value | humanizePercentage }}
|
|
412
|
+
LABELS = {{ $labels }}
|