sensu-plugins-kubernetes-reactiveops 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +50 -0
- data/LICENSE +22 -0
- data/README.md +229 -0
- data/bin/check-kube-apiserver-available.rb +56 -0
- data/bin/check-kube-nodes-ready.rb +59 -0
- data/bin/check-kube-pods-pending.rb +116 -0
- data/bin/check-kube-pods-restarting.rb +116 -0
- data/bin/check-kube-pods-running.rb +104 -0
- data/bin/check-kube-pods-runtime.rb +116 -0
- data/bin/check-kube-service-available.rb +121 -0
- data/bin/handler-kube-pod.rb +87 -0
- data/bin/metrics-pods.rb +59 -0
- data/lib/sensu-plugins-kubernetes.rb +1 -0
- data/lib/sensu-plugins-kubernetes/cli.rb +96 -0
- data/lib/sensu-plugins-kubernetes/client.rb +126 -0
- data/lib/sensu-plugins-kubernetes/version.rb +9 -0
- metadata +242 -0
@@ -0,0 +1,116 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# check-kube-pods-restarting
|
4
|
+
#
|
5
|
+
# DESCRIPTION:
|
6
|
+
# => Check if pods are constantly restarting
|
7
|
+
#
|
8
|
+
# OUTPUT:
|
9
|
+
# plain text
|
10
|
+
#
|
11
|
+
# PLATFORMS:
|
12
|
+
# Linux
|
13
|
+
#
|
14
|
+
# DEPENDENCIES:
|
15
|
+
# gem: sensu-plugin
|
16
|
+
# gem: kube-client
|
17
|
+
#
|
18
|
+
# USAGE:
|
19
|
+
# --ca-file CA-FILE CA file to verify API server cert
|
20
|
+
# --cert CERT-FILE Client cert to present
|
21
|
+
# --key KEY-FILE Client key for the client cert
|
22
|
+
# --in-cluster Use service account authentication
|
23
|
+
# --password PASSWORD If user is passed, also pass a password
|
24
|
+
# -s, --api-server URL URL to API server
|
25
|
+
# -t, --token TOKEN Bearer token for authorization
|
26
|
+
# --token-file TOKEN-FILE File containing bearer token for authorization
|
27
|
+
# -u, --user USER User with access to API
|
28
|
+
# -v, --api-version VERSION API version
|
29
|
+
# -n NAMESPACES, Exclude the specified list of namespaces
|
30
|
+
# --exclude-namespace
|
31
|
+
# -f, --filter FILTER Selector filter for pods to be checked
|
32
|
+
# -p, --pods PODS List of pods to check
|
33
|
+
# -r, --restart COUNT Threshold for number of restarts allowed
|
34
|
+
#
|
35
|
+
# NOTES:
|
36
|
+
# => The filter used for the -f flag is in the form key=value. If multiple
|
37
|
+
# filters need to be specfied, use a comma. ex. foo=bar,red=color
|
38
|
+
#
|
39
|
+
# LICENSE:
|
40
|
+
# Barry Martin <nyxcharon@gmail.com>
|
41
|
+
# Released under the same terms as Sensu (the MIT license); see LICENSE
|
42
|
+
# for details.
|
43
|
+
#
|
44
|
+
|
45
|
+
require 'sensu-plugins-kubernetes/cli'
|
46
|
+
|
47
|
+
class PodsRestarting < Sensu::Plugins::Kubernetes::CLI
|
48
|
+
@options = Sensu::Plugins::Kubernetes::CLI.options.dup
|
49
|
+
|
50
|
+
option :pod_list,
|
51
|
+
description: 'List of pods to check',
|
52
|
+
short: '-p PODS',
|
53
|
+
long: '--pods',
|
54
|
+
default: 'all'
|
55
|
+
|
56
|
+
option :restart_count,
|
57
|
+
description: 'Threshold for number of restarts allowed',
|
58
|
+
short: '-r COUNT',
|
59
|
+
long: '--restart',
|
60
|
+
proc: proc(&:to_i),
|
61
|
+
default: 10
|
62
|
+
|
63
|
+
option :pod_filter,
|
64
|
+
description: 'Selector filter for pods to be checked',
|
65
|
+
short: '-f FILTER',
|
66
|
+
long: '--filter'
|
67
|
+
|
68
|
+
option :exclude_namespace,
|
69
|
+
description: 'Exclude the specified list of namespaces',
|
70
|
+
short: '-n NAMESPACES',
|
71
|
+
long: '--exclude-namespace',
|
72
|
+
proc: proc { |a| a.split(',') },
|
73
|
+
default: ''
|
74
|
+
|
75
|
+
def run
|
76
|
+
pods_list = []
|
77
|
+
restarted_pods = []
|
78
|
+
pods = []
|
79
|
+
if config[:pod_filter].nil?
|
80
|
+
pods_list = parse_list(config[:pod_list])
|
81
|
+
pods = client.get_pods
|
82
|
+
else
|
83
|
+
pods = client.get_pods(label_selector: config[:pod_filter].to_s)
|
84
|
+
if pods.empty?
|
85
|
+
unknown 'The filter specified resulted in 0 pods'
|
86
|
+
end
|
87
|
+
pods_list = ['all']
|
88
|
+
end
|
89
|
+
pods.each do |pod|
|
90
|
+
next if pod.nil?
|
91
|
+
next if config[:exclude_namespace].include?(pod.metadata.namespace)
|
92
|
+
next unless pods_list.include?(pod.metadata.name) || pods_list.include?('all')
|
93
|
+
# Check restarts
|
94
|
+
next if pod.status.containerStatuses.nil?
|
95
|
+
pod.status.containerStatuses.each do |container|
|
96
|
+
if container.restartCount.to_i > config[:restart_count]
|
97
|
+
restarted_pods << "#{pod.metadata.namespace}.#{container.name}"
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
if restarted_pods.empty?
|
103
|
+
ok 'All pods are under restart threshold'
|
104
|
+
else
|
105
|
+
critical "Pods exceeded restart threshold: #{restarted_pods.join(' ')}"
|
106
|
+
end
|
107
|
+
rescue KubeException => e
|
108
|
+
critical 'API error: ' << e.message
|
109
|
+
end
|
110
|
+
|
111
|
+
def parse_list(list)
|
112
|
+
return list.split(',') if list && list.include?(',')
|
113
|
+
return [list] if list
|
114
|
+
['']
|
115
|
+
end
|
116
|
+
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# check-kube-pods-running
|
4
|
+
#
|
5
|
+
# DESCRIPTION:
|
6
|
+
# => Check if pods are in a running state
|
7
|
+
#
|
8
|
+
# OUTPUT:
|
9
|
+
# plain text
|
10
|
+
#
|
11
|
+
# PLATFORMS:
|
12
|
+
# Linux
|
13
|
+
#
|
14
|
+
# DEPENDENCIES:
|
15
|
+
# gem: sensu-plugin
|
16
|
+
# gem: kube-client
|
17
|
+
#
|
18
|
+
# Usage: ./check-kube-pods-running.rb (options)
|
19
|
+
# --ca-file CA-FILE CA file to verify API server cert
|
20
|
+
# --cert CERT-FILE Client cert to present
|
21
|
+
# --key KEY-FILE Client key for the client cert
|
22
|
+
# --in-cluster Use service account authentication
|
23
|
+
# --password PASSWORD If user is passed, also pass a password
|
24
|
+
# -s, --api-server URL URL to API server
|
25
|
+
# -t, --token TOKEN Bearer token for authorization
|
26
|
+
# --token-file TOKEN-FILE File containing bearer token for authorization
|
27
|
+
# -u, --user USER User with access to API
|
28
|
+
# -v, --api-version VERSION API version
|
29
|
+
# -n NAMESPACES, Exclude the specified list of namespaces
|
30
|
+
# --exclude-namespace
|
31
|
+
# -f, --filter FILTER Selector filter for pods to be checked
|
32
|
+
# -p, --pods PODS List of pods to check
|
33
|
+
# NOTES:
|
34
|
+
# => The filter used for the -f flag is in the form key=value. If multiple
|
35
|
+
# filters need to be specfied, use a comma. ex. foo=bar,red=color
|
36
|
+
#
|
37
|
+
# LICENSE:
|
38
|
+
# Barry Martin <nyxcharon@gmail.com>
|
39
|
+
# Released under the same terms as Sensu (the MIT license); see LICENSE
|
40
|
+
# for details.
|
41
|
+
#
|
42
|
+
|
43
|
+
require 'sensu-plugins-kubernetes/cli'
|
44
|
+
|
45
|
+
class AllPodsAreRunning < Sensu::Plugins::Kubernetes::CLI
|
46
|
+
@options = Sensu::Plugins::Kubernetes::CLI.options.dup
|
47
|
+
|
48
|
+
option :pod_list,
|
49
|
+
description: 'List of pods to check',
|
50
|
+
short: '-p PODS',
|
51
|
+
long: '--pods',
|
52
|
+
default: 'all'
|
53
|
+
|
54
|
+
option :pod_filter,
|
55
|
+
description: 'Selector filter for pods to be checked',
|
56
|
+
short: '-f FILTER',
|
57
|
+
long: '--filter'
|
58
|
+
|
59
|
+
option :exclude_namespace,
|
60
|
+
description: 'Exclude the specified list of namespaces',
|
61
|
+
short: '-n NAMESPACES',
|
62
|
+
long: '--exclude-namespace',
|
63
|
+
proc: proc { |a| a.split(',') },
|
64
|
+
default: ''
|
65
|
+
|
66
|
+
def run
|
67
|
+
pods_list = []
|
68
|
+
failed_pods = []
|
69
|
+
pods = []
|
70
|
+
if config[:pod_filter].nil?
|
71
|
+
pods_list = parse_list(config[:pod_list])
|
72
|
+
pods = client.get_pods
|
73
|
+
else
|
74
|
+
pods = client.get_pods(label_selector: config[:pod_filter].to_s)
|
75
|
+
if pods.empty?
|
76
|
+
unknown 'The filter specified resulted in 0 pods'
|
77
|
+
end
|
78
|
+
pods_list = ['all']
|
79
|
+
end
|
80
|
+
pods.each do |pod|
|
81
|
+
next if pod.nil?
|
82
|
+
next if config[:exclude_namespace].include?(pod.metadata.namespace)
|
83
|
+
next unless pods_list.include?(pod.metadata.name) || pods_list.include?('all')
|
84
|
+
next unless pod.status.phase != 'Succeeded' && !pod.status.conditions.nil?
|
85
|
+
if pod.status.conditions[1].status == 'False' # This is the Ready state
|
86
|
+
failed_pods << pod.metadata.name
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
if failed_pods.empty?
|
91
|
+
ok 'All pods are reporting as ready'
|
92
|
+
else
|
93
|
+
critical "Pods found in a non-ready state: #{failed_pods.join(' ')}"
|
94
|
+
end
|
95
|
+
rescue KubeException => e
|
96
|
+
critical 'API error: ' << e.message
|
97
|
+
end
|
98
|
+
|
99
|
+
def parse_list(list)
|
100
|
+
return list.split(',') if list && list.include?(',')
|
101
|
+
return [list] if list
|
102
|
+
['']
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# check-kube-pods-runtime
|
4
|
+
#
|
5
|
+
# DESCRIPTION:
|
6
|
+
# => Check if pods are running longer than expected
|
7
|
+
#
|
8
|
+
# OUTPUT:
|
9
|
+
# plain text
|
10
|
+
#
|
11
|
+
# PLATFORMS:
|
12
|
+
# Linux
|
13
|
+
#
|
14
|
+
# DEPENDENCIES:
|
15
|
+
# gem: sensu-plugin
|
16
|
+
# gem: kube-client
|
17
|
+
#
|
18
|
+
# USAGE:
|
19
|
+
# -s, --api-server URL URL to API server
|
20
|
+
# -v, --api-version VERSION API version. Defaults to 'v1'
|
21
|
+
# --in-cluster Use service account authentication
|
22
|
+
# --ca-file CA-FILE CA file to verify API server cert
|
23
|
+
# --cert CERT-FILE Client cert to present
|
24
|
+
# --key KEY-FILE Client key for the client cert
|
25
|
+
# -u, --user USER User with access to API
|
26
|
+
# --password PASSWORD If user is passed, also pass a password
|
27
|
+
# --token TOKEN Bearer token for authorization
|
28
|
+
# --token-file TOKEN-FILE File containing bearer token for authorization
|
29
|
+
# -c, --critical COUNT Threshold for Pods to be critical
|
30
|
+
# -f, --filter FILTER Selector filter for pods to be checked
|
31
|
+
# -p, --pods PODS List of pods to check
|
32
|
+
# -w, --warn TIMEOUT Threshold for pods to be in the pending state
|
33
|
+
#
|
34
|
+
# LICENSE:
|
35
|
+
# Barry Martin <nyxcharon@gmail.com>
|
36
|
+
# Released under the same terms as Sensu (the MIT license); see LICENSE
|
37
|
+
# for details.
|
38
|
+
#
|
39
|
+
|
40
|
+
require 'sensu-plugins-kubernetes/cli'
|
41
|
+
|
42
|
+
class PodRuntime < Sensu::Plugins::Kubernetes::CLI
|
43
|
+
@options = Sensu::Plugins::Kubernetes::CLI.options.dup
|
44
|
+
|
45
|
+
option :pod_list,
|
46
|
+
description: 'List of pods to check',
|
47
|
+
short: '-p PODS',
|
48
|
+
long: '--pods',
|
49
|
+
default: 'all'
|
50
|
+
|
51
|
+
option :pod_filter,
|
52
|
+
description: 'Selector filter for pods to be checked',
|
53
|
+
short: '-f FILTER',
|
54
|
+
long: '--filter'
|
55
|
+
|
56
|
+
option :warn_timeout,
|
57
|
+
description: 'Threshold for pods to be in the pending state',
|
58
|
+
short: '-w TIMEOUT',
|
59
|
+
long: '--warn',
|
60
|
+
proc: proc(&:to_i)
|
61
|
+
|
62
|
+
option :critical_timeout,
|
63
|
+
description: 'Threshold for Pods to be critical',
|
64
|
+
short: '-c COUNT',
|
65
|
+
long: '--critical',
|
66
|
+
proc: proc(&:to_i)
|
67
|
+
|
68
|
+
def run
|
69
|
+
pods_list = []
|
70
|
+
pods = []
|
71
|
+
warn = false
|
72
|
+
crit = false
|
73
|
+
message = ''
|
74
|
+
|
75
|
+
if config[:pod_filter].nil?
|
76
|
+
pods_list = parse_list(config[:pod_list])
|
77
|
+
pods = client.get_pods
|
78
|
+
else
|
79
|
+
pods = client.get_pods(label_selector: config[:pod_filter].to_s)
|
80
|
+
pods_list = ['all']
|
81
|
+
end
|
82
|
+
|
83
|
+
pods.each do |pod|
|
84
|
+
next if pod.nil?
|
85
|
+
next unless pods_list.include?(pod.metadata.name) || pods_list.include?('all')
|
86
|
+
# Check for Running state
|
87
|
+
next unless pod.status.phase == 'Running'
|
88
|
+
pod_stamp = Time.parse(pod.status.startTime)
|
89
|
+
runtime = (Time.now.utc - pod_stamp.utc).to_i
|
90
|
+
|
91
|
+
if !config[:critical_timeout].nil? && runtime > config[:critical_timeout]
|
92
|
+
message << "#{pod.metadata.name} exceeds threshold #{config[:critical_timeout]} "
|
93
|
+
crit = true
|
94
|
+
elsif !config[:warn_timeout].nil? && runtime > config[:warn_timeout]
|
95
|
+
message << "#{pod.metadata.name} exceeds threshold #{config[:warn_timeout]} "
|
96
|
+
warn = true
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
if crit
|
101
|
+
critical message
|
102
|
+
elsif warn
|
103
|
+
warning message
|
104
|
+
else
|
105
|
+
ok 'All pods within threshold'
|
106
|
+
end
|
107
|
+
rescue KubeException => e
|
108
|
+
critical 'API error: ' << e.message
|
109
|
+
end
|
110
|
+
|
111
|
+
def parse_list(list)
|
112
|
+
return list.split(',') if list && list.include?(',')
|
113
|
+
return [list] if list
|
114
|
+
['']
|
115
|
+
end
|
116
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# check-kube-pods-service-available
|
4
|
+
#
|
5
|
+
# DESCRIPTION:
|
6
|
+
# => Check if your kube services are up and ready
|
7
|
+
#
|
8
|
+
# OUTPUT:
|
9
|
+
# plain text
|
10
|
+
#
|
11
|
+
# PLATFORMS:
|
12
|
+
# Linux
|
13
|
+
#
|
14
|
+
# DEPENDENCIES:
|
15
|
+
# gem: sensu-plugin
|
16
|
+
# gem: kube-client
|
17
|
+
#
|
18
|
+
# USAGE:
|
19
|
+
# -s, --api-server URL URL to API server
|
20
|
+
# -v, --api-version VERSION API version. Defaults to 'v1'
|
21
|
+
# --in-cluster Use service account authentication
|
22
|
+
# --ca-file CA-FILE CA file to verify API server cert
|
23
|
+
# --cert CERT-FILE Client cert to present
|
24
|
+
# --key KEY-FILE Client key for the client cert
|
25
|
+
# -u, --user USER User with access to API
|
26
|
+
# --password PASSWORD If user is passed, also pass a password
|
27
|
+
# --token TOKEN Bearer token for authorization
|
28
|
+
# --token-file TOKEN-FILE File containing bearer token for authorization
|
29
|
+
# -l, --list SERVICES List of services to check (required)
|
30
|
+
# -p, --pending SECONDS Time (in seconds) a pod may be pending for and be valid
|
31
|
+
#
|
32
|
+
# NOTES:
|
33
|
+
#
|
34
|
+
# LICENSE:
|
35
|
+
# Barry Martin <nyxcharon@gmail.com>
|
36
|
+
# Released under the same terms as Sensu (the MIT license); see LICENSE
|
37
|
+
# for details.
|
38
|
+
#
|
39
|
+
|
40
|
+
require 'sensu-plugins-kubernetes/cli'
|
41
|
+
require 'time'
|
42
|
+
|
43
|
+
class AllServicesUp < Sensu::Plugins::Kubernetes::CLI
|
44
|
+
@options = Sensu::Plugins::Kubernetes::CLI.options.dup
|
45
|
+
|
46
|
+
option :service_list,
|
47
|
+
description: 'List of services to check',
|
48
|
+
short: '-l SERVICES',
|
49
|
+
long: '--list',
|
50
|
+
required: true
|
51
|
+
|
52
|
+
option :pendingTime,
|
53
|
+
description: 'Time (in seconds) a pod may be pending for and be valid',
|
54
|
+
short: '-p SECONDS',
|
55
|
+
long: '--pending',
|
56
|
+
default: 0,
|
57
|
+
proc: proc(&:to_i)
|
58
|
+
|
59
|
+
def run
|
60
|
+
services = parse_list(config[:service_list])
|
61
|
+
failed_services = []
|
62
|
+
s = client.get_services
|
63
|
+
s.each do |a|
|
64
|
+
next unless services.include?(a.metadata.name)
|
65
|
+
# Build the selector key so we can fetch the corresponding pod
|
66
|
+
selector_key = []
|
67
|
+
services.delete(a.metadata.name)
|
68
|
+
a.spec.selector.to_h.each do |k, v|
|
69
|
+
selector_key << "#{k}=#{v}"
|
70
|
+
end
|
71
|
+
# Get the pod
|
72
|
+
pod = nil
|
73
|
+
begin
|
74
|
+
pod = client.get_pods(label_selector: selector_key.join(',').to_s)
|
75
|
+
rescue
|
76
|
+
failed_services << a.metadata.name.to_s
|
77
|
+
end
|
78
|
+
# Make sure our pod is running
|
79
|
+
next if pod.nil?
|
80
|
+
pod_available = false
|
81
|
+
pod.each do |p|
|
82
|
+
case p.status.phase
|
83
|
+
when 'Pending'
|
84
|
+
next if p.status.startTime.nil?
|
85
|
+
if (Time.now - Time.parse(p.status.startTime)).to_i < config[:pendingTime]
|
86
|
+
pod_available = true
|
87
|
+
break
|
88
|
+
end
|
89
|
+
when 'Running'
|
90
|
+
p.status.conditions.each do |c|
|
91
|
+
next unless c.type == 'Ready'
|
92
|
+
if c.status == 'True'
|
93
|
+
pod_available = true
|
94
|
+
break
|
95
|
+
end
|
96
|
+
break if pod_available
|
97
|
+
end
|
98
|
+
end
|
99
|
+
failed_services << "#{p.metadata.namespace}.#{p.metadata.name}" if pod_available == false
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
if failed_services.empty? && services.empty?
|
104
|
+
ok 'All services are reporting as up'
|
105
|
+
end
|
106
|
+
|
107
|
+
if !failed_services.empty?
|
108
|
+
critical "All services are not ready: #{failed_services.join(' ')}"
|
109
|
+
else
|
110
|
+
critical "Some services could not be checked: #{services.join(' ')}"
|
111
|
+
end
|
112
|
+
rescue KubeException => e
|
113
|
+
critical 'API error: ' << e.message
|
114
|
+
end
|
115
|
+
|
116
|
+
def parse_list(list)
|
117
|
+
return list.split(',') if list && list.include?(',')
|
118
|
+
return [list] if list
|
119
|
+
['']
|
120
|
+
end
|
121
|
+
end
|