@leverege/build-tools 2.66.1 → 2.66.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/package.json +2 -2
  2. package/src/bash-funcs +1 -0
  3. package/src/helm-charts/cnpg-db-pgbench/cluster.yaml.ovh +2 -0
  4. package/src/helm-charts/cnpg-db-tsdb-basic/cluster.yaml.ovh +3 -1
  5. package/src/helm-charts/cnpg-db-tsdb-dense/cluster.yaml.ovh +2 -0
  6. package/src/helm-charts/elasticsearch8/helmup.plugin +1 -1
  7. package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh +1 -1
  8. package/src/helm-charts/prom-operator/rules/cadvisor-rules.yaml +3 -3
  9. package/src/helm-charts/prom-operator/rules/cnpg-rules.yaml +3 -2
  10. package/src/helm-charts/prom-operator/rules/elasticsearch-rules.yaml +1 -1
  11. package/src/helm-charts/prom-operator/rules/gcp-rules.yaml +2 -2
  12. package/src/helm-charts/prom-operator/rules/kubestate-rules.yaml +1 -1
  13. package/src/helm-charts/prom-operator/rules/po-k8s-selective.yaml +23 -0
  14. package/src/helm-charts/prom-operator/rules/prometheus-rules.yaml +1 -1
  15. package/src/helm-charts/prom-operator/rules/redis-rules.yaml +1 -1
  16. package/src/helm-charts/prom-operator/rules/traefik-rules.yaml +2 -2
  17. package/src/helm-charts/prom-operator/update-rules.sh +32 -0
  18. package/src/helm-charts/redis/helmup.plugin +1 -1
  19. package/src/helm-charts/velero/helmup.plugin +1 -1
  20. package/src/helmup.sh +1 -1
  21. package/src/awesome-fetch.mjs +0 -44
  22. package/src/helm-charts/prom-operator/apply-rules +0 -15
  23. package/src/helm-charts/prom-operator/gitignore +0 -2
  24. package/src/helm-charts/prom-operator/rules/awesome-base/etcd-exporter.yaml +0 -153
  25. package/src/helm-charts/prom-operator/rules/etcd-rulte.yaml +0 -153
  26. /package/src/helm-charts/prom-operator/rules/{postgres-rules.yaml → postgres-rules.yaml.disabled} +0 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@leverege/build-tools",
3
- "version": "2.66.1",
3
+ "version": "2.66.2",
4
4
  "description": "A collection of build / support tools for Leverege developers",
5
5
  "main": "index.js",
6
6
  "repository": {
@@ -83,7 +83,7 @@
83
83
  "semver": "^7.7.1",
84
84
  "simple-git": "^3.27.0",
85
85
  "toml": "^3.0.0",
86
- "zx": "^8.5.2"
86
+ "zx": "^8.5.3"
87
87
  },
88
88
  "devDependencies": {
89
89
  "@leverege/eslint-config-leverege": "^5.0.1",
package/src/bash-funcs CHANGED
@@ -11,6 +11,7 @@
11
11
  DEVNULL="/dev/null"
12
12
  [ ! -z "$BUILD_TOOLS_DEBUG" ] && DEVNULL="/dev/stdout" && printf "\n***BUILD_TOOLS_DEBUG DEVNULL=>$DEVNULL\n"
13
13
 
14
+ GITTOP=`git rev-parse --show-toplevel`
14
15
  DTS="`date +\"%Y%m%d-%H%M%S\"`"
15
16
 
16
17
  # Simple ANSI color coded string generator.
@@ -14,6 +14,7 @@ spec:
14
14
  postInitTemplateSQL:
15
15
  - CREATE EXTENSION timescaledb;
16
16
  - CREATE EXTENSION jsquery;
17
+ - CREATE EXTENSION pg_stat_statements;
17
18
  database: imagine
18
19
  owner: imagine
19
20
  secret:
@@ -38,6 +39,7 @@ spec:
38
39
  shared_preload_libraries:
39
40
  - timescaledb
40
41
  - jsquery
42
+ - pg_stat_statements
41
43
  parameters:
42
44
  # max_worker_processes: "60"
43
45
  max_worker_processes: "8"
@@ -18,7 +18,8 @@ spec:
18
18
  postInitTemplateSQL:
19
19
  - CREATE EXTENSION timescaledb;
20
20
  - CREATE EXTENSION jsquery;
21
- - CREATE DATABASE imagine
21
+ - CREATE DATABASE pg_stat_statements;
22
+ - CREATE DATABASE imagine;
22
23
  # - CREATE DATABASE app; # for the operator restores
23
24
  # - CREATE ROLE app; # same
24
25
  # owner: imagine
@@ -52,6 +53,7 @@ spec:
52
53
  shared_preload_libraries:
53
54
  - timescaledb
54
55
  - jsquery
56
+ - pg_stat_statements
55
57
  parameters:
56
58
  # max_connections: "100"
57
59
  # max_worker_processes: "60"
@@ -17,6 +17,7 @@ spec:
17
17
  postInitTemplateSQL:
18
18
  - CREATE EXTENSION timescaledb;
19
19
  - CREATE EXTENSION jsquery;
20
+ - CREATE EXTENSION pg_stat_statements;
20
21
  - CREATE DATABASE imagine; # leverege tsdb db
21
22
  # secret:
22
23
  # name: cnpg-db-tsdb-dense-imagine-pw # imagine db pw
@@ -48,6 +49,7 @@ spec:
48
49
  shared_preload_libraries:
49
50
  - timescaledb
50
51
  - jsquery
52
+ - pg_stat_statements
51
53
  parameters:
52
54
  # max_worker_processes: "60"
53
55
  max_worker_processes: "8"
@@ -2,7 +2,7 @@
2
2
  #
3
3
  showInstalling "Elastic Search 8"
4
4
 
5
- [ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.5.0"
5
+ [ -z "$ELASTIC_CHART_VERSION" ] && ELASTIC_CHART_VERSION="21.6.0"
6
6
 
7
7
  OCI_CHART="oci://registry-1.docker.io/bitnamicharts/elasticsearch"
8
8
 
@@ -10,7 +10,7 @@ defaultRules:
10
10
  etcd: false
11
11
  configReloaders: false
12
12
  general: true
13
- k8s: true
13
+ k8s: false # GPT see po-k8s-selective
14
14
  kubeApiserver: true # GPT Keep an eye on API server availability
15
15
  kubeApiserverAvailability: false
16
16
  kubeApiserverSlos: false
@@ -1,8 +1,8 @@
1
- # https://samber.github.io/awesome-prometheus-alerts/rules#docker-containers
1
+ #https://samber.github.io/awesome-prometheus-alerts/rules#docker-containers
2
2
  apiVersion: monitoring.coreos.com/v1
3
3
  kind: PrometheusRule
4
4
  metadata:
5
- name: google-cadvisor
5
+ name: cadvisor-rules
6
6
  namespace: prometheus
7
7
  spec:
8
8
  groups:
@@ -10,7 +10,7 @@ spec:
10
10
  rules:
11
11
  - alert: ContainerKilled
12
12
  expr: time() - container_last_seen > 60
13
- for: 0m
13
+ for: 5m
14
14
  labels:
15
15
  severity: warning
16
16
  annotations:
@@ -1,10 +1,11 @@
1
+ # https://github.com/cloudnative-pg/cloudnative-pg/blob/main/docs/src/samples/monitoring/prometheusrule.yaml
1
2
  apiVersion: monitoring.coreos.com/v1
2
3
  kind: PrometheusRule
3
4
  metadata:
4
- name: cnpg-default-alerts
5
+ name: cnpg-rules
5
6
  spec:
6
7
  groups:
7
- - name: cnpg-default.rules
8
+ - name: CNPG
8
9
  rules:
9
10
  - alert: LongRunningTransaction
10
11
  annotations:
@@ -2,7 +2,7 @@
2
2
  apiVersion: monitoring.coreos.com/v1
3
3
  kind: PrometheusRule
4
4
  metadata:
5
- name: prometheus-community-elasticsearch-exporter
5
+ name: elasticsearch-rules
6
6
  namespace: prometheus
7
7
  spec:
8
8
  groups:
@@ -1,12 +1,12 @@
1
1
  apiVersion: monitoring.coreos.com/v1
2
2
  kind: PrometheusRule
3
3
  metadata:
4
- name: gcp-rules
4
+ name: stackdriver-rules
5
5
  namespace: prometheus
6
6
 
7
7
  spec:
8
8
  groups:
9
- - name: GCP
9
+ - name: Stackdriver
10
10
  rules:
11
11
  - alert: StackdriverFailedScrape
12
12
  expr: absent(stackdriver_pubsub_topic_pubsub_googleapis_com_topic_message_sizes_count) == 1
@@ -2,7 +2,7 @@
2
2
  apiVersion: monitoring.coreos.com/v1
3
3
  kind: PrometheusRule
4
4
  metadata:
5
- name: kubestate-exporter
5
+ name: kubestate-rules
6
6
  namespace: prometheus
7
7
  spec:
8
8
  groups:
@@ -0,0 +1,23 @@
1
+ # selected alerts from prom-operator default k8s rules (ChatGPT recommendation)
2
+ apiVersion: monitoring.coreos.com/v1
3
+ kind: PrometheusRule
4
+ metadata:
5
+ name: po-k8s-selective-alerts
6
+ namespace: prometheus
7
+ labels:
8
+ role: alert-rules
9
+ prometheus: kube-prometheus
10
+ spec:
11
+ groups:
12
+ - name: PromOp K8s Selective
13
+ rules:
14
+ - alert: KubePodImagePullBackOff
15
+ expr: kube_pod_container_status_waiting_reason{reason="ImagePullBackOff", namespace!~"kube-system|prometheus"} > 0
16
+ for: 2m
17
+ labels:
18
+ severity: warning
19
+ annotations:
20
+ summary: Pod has image pull back-off ({{ $labels.namespace }}/{{ $labels.pod }})
21
+ description: Pod '{{ $labels.pod }}' in namespace '{{ $labels.namespace }}' has containers in ImagePullBackOff.
22
+ VALUE = {{ $value }}
23
+ LABELS = {{ $labels }}
@@ -2,7 +2,7 @@
2
2
  apiVersion: monitoring.coreos.com/v1
3
3
  kind: PrometheusRule
4
4
  metadata:
5
- name: prometheus-self-monitoring # embedded-exporter
5
+ name: prometheus-rules # embedded-exporter
6
6
  namespace: prometheus
7
7
  spec:
8
8
  groups:
@@ -2,7 +2,7 @@
2
2
  apiVersion: monitoring.coreos.com/v1
3
3
  kind: PrometheusRule
4
4
  metadata:
5
- name: redis
5
+ name: redis-rules
6
6
  namespace: prometheus
7
7
  spec:
8
8
  groups:
@@ -2,11 +2,11 @@
2
2
  apiVersion: monitoring.coreos.com/v1
3
3
  kind: PrometheusRule
4
4
  metadata:
5
- name: embedded-exporter-v2
5
+ name: traefik-rules
6
6
  namespace: prometheus
7
7
  spec:
8
8
  groups:
9
- - name: EmbeddedExporterV2-rules
9
+ - name: Traefik (awesome) # EmbeddedExporterV2-rules
10
10
  rules:
11
11
  - alert: TraefikServiceDown
12
12
  expr: count(traefik_service_server_up) by (service) == 0
@@ -0,0 +1,32 @@
1
+ #!/bin/bash
2
+ #
3
+ # Load the standard helper functions
4
+ . `build-tools --bashfun`
5
+
6
+ RULES_DIR="$GITTOP/prom-operator/rules"
7
+ [ ! -d "$RULES_DIR" ] && errorExit "Missing rules directory [$RULES_DIR]"
8
+
9
+ case "$1" in
10
+ "apply"|"delete")
11
+ cd $GITTOP
12
+ overwhelm
13
+ for ruleset in $RULES_DIR/*.yaml; do
14
+ shortName=`basename $ruleset`
15
+ printf "\n $1 rules => `color g $shortName`\n"
16
+ kubectl $1 -n prometheus -f $ruleset
17
+ done
18
+ echo "" && kubectl get -A prometheusrules.monitoring.coreos.com
19
+ ;;
20
+
21
+ *)
22
+ cat<<USAGE_HELP
23
+
24
+ Usage: $0 [apply|delete]
25
+
26
+ This will either apply or delete the rules defined in $RULES_DIR
27
+
28
+ USAGE_HELP
29
+
30
+ exit 1
31
+ ;;
32
+ esac
@@ -4,7 +4,7 @@ showInstalling "Redis"
4
4
 
5
5
  OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
6
6
 
7
- [ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.11.5"
7
+ [ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.12.1"
8
8
 
9
9
  helm upgrade --install redis $OCI_CHART \
10
10
  --values redis/values.yaml \
@@ -4,7 +4,7 @@ showInstalling "Velero Backup System"
4
4
 
5
5
  addHelmRepo vmware-tanzu https://vmware-tanzu.github.io/helm-charts
6
6
 
7
- [ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="8"
7
+ [ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="9"
8
8
  #
9
9
  # Build the bucket, region and SA email variables and use --set to mod the
10
10
  # chart values as opposed to using an OVH:<label> approach. This eliminates
package/src/helmup.sh CHANGED
@@ -791,7 +791,7 @@ function installVeleroEnvironment() {
791
791
  gcloud config set project $GCP_PROJECT_ID
792
792
 
793
793
  ## The major chart version will determine the bucket suffix
794
- [ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="8"
794
+ [ -z "$VELERO_CHART_VERSION" ] && VELERO_CHART_VERSION="9"
795
795
 
796
796
  ## Create a bucket
797
797
  BUCKET="$GCP_PROJECT_ID-velero-$VELERO_CHART_VERSION"
@@ -1,44 +0,0 @@
1
- #!/usr/bin/env node
2
-
3
- import fs from 'node:fs'
4
- import yaml from 'js-yaml'
5
-
6
- const SOURCE_URL = 'https://raw.githubusercontent.com/samber/awesome-prometheus-alerts/master/dist/rules/kubernetes/kubestate-exporter.yml'
7
- const OUTPUT_FILE = 'converted-alerts.yaml'
8
-
9
- async function fetchAndConvert() {
10
- /* eslint-disable no-console */
11
- try {
12
- console.log( `Fetching YAML from ${SOURCE_URL}...` )
13
- const response = await fetch( SOURCE_URL )
14
- if ( !response.ok ) throw new Error( `Failed to fetch: ${response.statusText}` )
15
-
16
- const rawYaml = await response.text()
17
- const parsedYaml = yaml.load( rawYaml )
18
- if ( !parsedYaml.groups ) throw new Error( 'Invalid YAML structure, missing "groups"' )
19
-
20
- // Reformat into PrometheusRule CRD format
21
- const prometheusRule = {
22
- apiVersion : 'monitoring.coreos.com/v1',
23
- kind : 'PrometheusRule',
24
- metadata : {
25
- name : 'node-alerts', // Adjust based on the source
26
- },
27
- spec : {
28
- groups : parsedYaml.groups.map( group => ( {
29
- name : `${group.name}-rules`,
30
- rules : group.rules,
31
- } ) ),
32
- },
33
- }
34
-
35
- // Convert back to YAML with proper formatting
36
- const formattedYaml = yaml.dump( prometheusRule, { noRefs : true, indent : 0 } )
37
- fs.writeFileSync( OUTPUT_FILE, formattedYaml, 'utf8' )
38
- console.log( `Converted YAML written to ${OUTPUT_FILE}` )
39
- } catch ( error ) {
40
- console.error( 'Error processing YAML:', error.message )
41
- }
42
- }
43
-
44
- fetchAndConvert()
@@ -1,15 +0,0 @@
1
- #!/bin/bash
2
- #
3
- # Load the standard helper functions
4
- . `build-tools --bashfun`
5
-
6
- overwhelm
7
-
8
- RULES_DIR="prom-operator/rules"
9
-
10
- [ ! -d "$RULES_DIR" ] && errorExit "Missing rules directory [$RULES_DIR]"
11
-
12
- for ruleset in $RULES_DIR/*.yaml; do
13
- printf " Applying rules => `color g $ruleset`\n"
14
- kubectl apply -n prometheus -f $ruleset
15
- done
@@ -1,2 +0,0 @@
1
- prometheus-stack.yaml
2
- stackdriver-exporter.yaml
@@ -1,153 +0,0 @@
1
- # https://samber.github.io/awesome-prometheus-alerts/rules#etcd
2
- apiVersion: monitoring.coreos.com/v1
3
- kind: PrometheusRule
4
- metadata:
5
- name: etcd-exporter # embedded-exporter
6
- namespace: prometheus
7
- spec:
8
- groups:
9
- - name: Etcd (awesome) # EmbeddedExporter-rules
10
- rules:
11
- - alert: EtcdInsufficientMembers
12
- expr: count(etcd_server_id) % 2 == 0
13
- for: 0m
14
- labels:
15
- severity: critical
16
- annotations:
17
- summary: Etcd insufficient Members (instance {{ $labels.instance }})
18
- description: |-
19
- Etcd cluster should have an odd number of members
20
- VALUE = {{ $value }}
21
- LABELS = {{ $labels }}
22
- - alert: EtcdNoLeader
23
- expr: etcd_server_has_leader == 0
24
- for: 0m
25
- labels:
26
- severity: critical
27
- annotations:
28
- summary: Etcd no Leader (instance {{ $labels.instance }})
29
- description: |-
30
- Etcd cluster have no leader
31
- VALUE = {{ $value }}
32
- LABELS = {{ $labels }}
33
- - alert: EtcdHighNumberOfLeaderChanges
34
- expr: increase(etcd_server_leader_changes_seen_total[10m]) > 2
35
- for: 0m
36
- labels:
37
- severity: warning
38
- annotations:
39
- summary: Etcd high number of leader changes (instance {{ $labels.instance }})
40
- description: |-
41
- Etcd leader changed more than 2 times during 10 minutes
42
- VALUE = {{ $value }}
43
- LABELS = {{ $labels }}
44
- - alert: EtcdHighNumberOfFailedGrpcRequests
45
- expr: sum(rate(grpc_server_handled_total{grpc_code!="OK"}[1m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[1m])) BY (grpc_service, grpc_method) > 0.01
46
- for: 2m
47
- labels:
48
- severity: warning
49
- annotations:
50
- summary: Etcd high number of failed GRPC requests (instance {{ $labels.instance }})
51
- description: |-
52
- More than 1% GRPC request failure detected in Etcd
53
- VALUE = {{ $value }}
54
- LABELS = {{ $labels }}
55
- - alert: EtcdHighNumberOfFailedGrpcRequests
56
- expr: sum(rate(grpc_server_handled_total{grpc_code!="OK"}[1m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[1m])) BY (grpc_service, grpc_method) > 0.05
57
- for: 2m
58
- labels:
59
- severity: critical
60
- annotations:
61
- summary: Etcd high number of failed GRPC requests (instance {{ $labels.instance }})
62
- description: |-
63
- More than 5% GRPC request failure detected in Etcd
64
- VALUE = {{ $value }}
65
- LABELS = {{ $labels }}
66
- - alert: EtcdGrpcRequestsSlow
67
- expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_type="unary"}[1m])) by (grpc_service, grpc_method, le)) > 0.15
68
- for: 2m
69
- labels:
70
- severity: warning
71
- annotations:
72
- summary: Etcd GRPC requests slow (instance {{ $labels.instance }})
73
- description: |-
74
- GRPC requests slowing down, 99th percentile is over 0.15s
75
- VALUE = {{ $value }}
76
- LABELS = {{ $labels }}
77
- - alert: EtcdHighNumberOfFailedHttpRequests
78
- expr: sum(rate(etcd_http_failed_total[1m])) BY (method) / sum(rate(etcd_http_received_total[1m])) BY (method) > 0.01
79
- for: 2m
80
- labels:
81
- severity: warning
82
- annotations:
83
- summary: Etcd high number of failed HTTP requests (instance {{ $labels.instance }})
84
- description: |-
85
- More than 1% HTTP failure detected in Etcd
86
- VALUE = {{ $value }}
87
- LABELS = {{ $labels }}
88
- - alert: EtcdHighNumberOfFailedHttpRequests
89
- expr: sum(rate(etcd_http_failed_total[1m])) BY (method) / sum(rate(etcd_http_received_total[1m])) BY (method) > 0.05
90
- for: 2m
91
- labels:
92
- severity: critical
93
- annotations:
94
- summary: Etcd high number of failed HTTP requests (instance {{ $labels.instance }})
95
- description: |-
96
- More than 5% HTTP failure detected in Etcd
97
- VALUE = {{ $value }}
98
- LABELS = {{ $labels }}
99
- - alert: EtcdHttpRequestsSlow
100
- expr: histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[1m])) > 0.15
101
- for: 2m
102
- labels:
103
- severity: warning
104
- annotations:
105
- summary: Etcd HTTP requests slow (instance {{ $labels.instance }})
106
- description: |-
107
- HTTP requests slowing down, 99th percentile is over 0.15s
108
- VALUE = {{ $value }}
109
- LABELS = {{ $labels }}
110
- - alert: EtcdMemberCommunicationSlow
111
- expr: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) > 0.15
112
- for: 2m
113
- labels:
114
- severity: warning
115
- annotations:
116
- summary: Etcd member communication slow (instance {{ $labels.instance }})
117
- description: |-
118
- Etcd member communication slowing down, 99th percentile is over 0.15s
119
- VALUE = {{ $value }}
120
- LABELS = {{ $labels }}
121
- - alert: EtcdHighNumberOfFailedProposals
122
- expr: increase(etcd_server_proposals_failed_total[1h]) > 5
123
- for: 2m
124
- labels:
125
- severity: warning
126
- annotations:
127
- summary: Etcd high number of failed proposals (instance {{ $labels.instance }})
128
- description: |-
129
- Etcd server got more than 5 failed proposals past hour
130
- VALUE = {{ $value }}
131
- LABELS = {{ $labels }}
132
- - alert: EtcdHighFsyncDurations
133
- expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) > 0.5
134
- for: 2m
135
- labels:
136
- severity: warning
137
- annotations:
138
- summary: Etcd high fsync durations (instance {{ $labels.instance }})
139
- description: |-
140
- Etcd WAL fsync duration increasing, 99th percentile is over 0.5s
141
- VALUE = {{ $value }}
142
- LABELS = {{ $labels }}
143
- - alert: EtcdHighCommitDurations
144
- expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[1m])) > 0.25
145
- for: 2m
146
- labels:
147
- severity: warning
148
- annotations:
149
- summary: Etcd high commit durations (instance {{ $labels.instance }})
150
- description: |-
151
- Etcd commit duration increasing, 99th percentile is over 0.25s
152
- VALUE = {{ $value }}
153
- LABELS = {{ $labels }}
@@ -1,153 +0,0 @@
1
- # https://samber.github.io/awesome-prometheus-alerts/rules#etcd
2
- apiVersion: monitoring.coreos.com/v1
3
- kind: PrometheusRule
4
- metadata:
5
- name: etcd-exporter # embedded-exporter
6
- namespace: prometheus
7
- spec:
8
- groups:
9
- - name: Etcd (awesome) # EmbeddedExporter-rules
10
- rules:
11
- - alert: EtcdInsufficientMembers
12
- expr: count(etcd_server_id) % 2 == 0
13
- for: 0m
14
- labels:
15
- severity: critical
16
- annotations:
17
- summary: Etcd insufficient Members (instance {{ $labels.instance }})
18
- description: |-
19
- Etcd cluster should have an odd number of members
20
- VALUE = {{ $value }}
21
- LABELS = {{ $labels }}
22
- - alert: EtcdNoLeader
23
- expr: etcd_server_has_leader == 0
24
- for: 0m
25
- labels:
26
- severity: critical
27
- annotations:
28
- summary: Etcd no Leader (instance {{ $labels.instance }})
29
- description: |-
30
- Etcd cluster have no leader
31
- VALUE = {{ $value }}
32
- LABELS = {{ $labels }}
33
- - alert: EtcdHighNumberOfLeaderChanges
34
- expr: increase(etcd_server_leader_changes_seen_total[10m]) > 2
35
- for: 0m
36
- labels:
37
- severity: warning
38
- annotations:
39
- summary: Etcd high number of leader changes (instance {{ $labels.instance }})
40
- description: |-
41
- Etcd leader changed more than 2 times during 10 minutes
42
- VALUE = {{ $value }}
43
- LABELS = {{ $labels }}
44
- - alert: EtcdHighNumberOfFailedGrpcRequests
45
- expr: sum(rate(grpc_server_handled_total{grpc_code!="OK"}[1m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[1m])) BY (grpc_service, grpc_method) > 0.01
46
- for: 2m
47
- labels:
48
- severity: warning
49
- annotations:
50
- summary: Etcd high number of failed GRPC requests (instance {{ $labels.instance }})
51
- description: |-
52
- More than 1% GRPC request failure detected in Etcd
53
- VALUE = {{ $value }}
54
- LABELS = {{ $labels }}
55
- - alert: EtcdHighNumberOfFailedGrpcRequests
56
- expr: sum(rate(grpc_server_handled_total{grpc_code!="OK"}[1m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[1m])) BY (grpc_service, grpc_method) > 0.05
57
- for: 2m
58
- labels:
59
- severity: critical
60
- annotations:
61
- summary: Etcd high number of failed GRPC requests (instance {{ $labels.instance }})
62
- description: |-
63
- More than 5% GRPC request failure detected in Etcd
64
- VALUE = {{ $value }}
65
- LABELS = {{ $labels }}
66
- - alert: EtcdGrpcRequestsSlow
67
- expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_type="unary"}[1m])) by (grpc_service, grpc_method, le)) > 0.15
68
- for: 2m
69
- labels:
70
- severity: warning
71
- annotations:
72
- summary: Etcd GRPC requests slow (instance {{ $labels.instance }})
73
- description: |-
74
- GRPC requests slowing down, 99th percentile is over 0.15s
75
- VALUE = {{ $value }}
76
- LABELS = {{ $labels }}
77
- - alert: EtcdHighNumberOfFailedHttpRequests
78
- expr: sum(rate(etcd_http_failed_total[1m])) BY (method) / sum(rate(etcd_http_received_total[1m])) BY (method) > 0.01
79
- for: 2m
80
- labels:
81
- severity: warning
82
- annotations:
83
- summary: Etcd high number of failed HTTP requests (instance {{ $labels.instance }})
84
- description: |-
85
- More than 1% HTTP failure detected in Etcd
86
- VALUE = {{ $value }}
87
- LABELS = {{ $labels }}
88
- - alert: EtcdHighNumberOfFailedHttpRequests
89
- expr: sum(rate(etcd_http_failed_total[1m])) BY (method) / sum(rate(etcd_http_received_total[1m])) BY (method) > 0.05
90
- for: 2m
91
- labels:
92
- severity: critical
93
- annotations:
94
- summary: Etcd high number of failed HTTP requests (instance {{ $labels.instance }})
95
- description: |-
96
- More than 5% HTTP failure detected in Etcd
97
- VALUE = {{ $value }}
98
- LABELS = {{ $labels }}
99
- - alert: EtcdHttpRequestsSlow
100
- expr: histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[1m])) > 0.15
101
- for: 2m
102
- labels:
103
- severity: warning
104
- annotations:
105
- summary: Etcd HTTP requests slow (instance {{ $labels.instance }})
106
- description: |-
107
- HTTP requests slowing down, 99th percentile is over 0.15s
108
- VALUE = {{ $value }}
109
- LABELS = {{ $labels }}
110
- - alert: EtcdMemberCommunicationSlow
111
- expr: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) > 0.15
112
- for: 2m
113
- labels:
114
- severity: warning
115
- annotations:
116
- summary: Etcd member communication slow (instance {{ $labels.instance }})
117
- description: |-
118
- Etcd member communication slowing down, 99th percentile is over 0.15s
119
- VALUE = {{ $value }}
120
- LABELS = {{ $labels }}
121
- - alert: EtcdHighNumberOfFailedProposals
122
- expr: increase(etcd_server_proposals_failed_total[1h]) > 5
123
- for: 2m
124
- labels:
125
- severity: warning
126
- annotations:
127
- summary: Etcd high number of failed proposals (instance {{ $labels.instance }})
128
- description: |-
129
- Etcd server got more than 5 failed proposals past hour
130
- VALUE = {{ $value }}
131
- LABELS = {{ $labels }}
132
- - alert: EtcdHighFsyncDurations
133
- expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) > 0.5
134
- for: 2m
135
- labels:
136
- severity: warning
137
- annotations:
138
- summary: Etcd high fsync durations (instance {{ $labels.instance }})
139
- description: |-
140
- Etcd WAL fsync duration increasing, 99th percentile is over 0.5s
141
- VALUE = {{ $value }}
142
- LABELS = {{ $labels }}
143
- - alert: EtcdHighCommitDurations
144
- expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[1m])) > 0.25
145
- for: 2m
146
- labels:
147
- severity: warning
148
- annotations:
149
- summary: Etcd high commit durations (instance {{ $labels.instance }})
150
- description: |-
151
- Etcd commit duration increasing, 99th percentile is over 0.25s
152
- VALUE = {{ $value }}
153
- LABELS = {{ $labels }}