sensu-plugins-kubernetes-reactiveops 0.1.0

data/bin/check-kube-pods-restarting.rb

@@ -0,0 +1,116 @@
+#! /usr/bin/env ruby
+#
+#   check-kube-pods-restarting
+#
+# DESCRIPTION:
+# => Check if pods are constantly restarting
+#
+# OUTPUT:
+#   plain text
+#
+# PLATFORMS:
+#   Linux
+#
+# DEPENDENCIES:
+#   gem: sensu-plugin
+#   gem: kube-client
+#
+# USAGE:
+# --ca-file CA-FILE            CA file to verify API server cert
+#        --cert CERT-FILE             Client cert to present
+#        --key KEY-FILE               Client key for the client cert
+#        --in-cluster                 Use service account authentication
+#        --password PASSWORD          If user is passed, also pass a password
+#    -s, --api-server URL             URL to API server
+#    -t, --token TOKEN                Bearer token for authorization
+#        --token-file TOKEN-FILE      File containing bearer token for authorization
+#    -u, --user USER                  User with access to API
+#    -v, --api-version VERSION        API version
+#    -n NAMESPACES,                   Exclude the specified list of namespaces
+#        --exclude-namespace
+#    -f, --filter FILTER              Selector filter for pods to be checked
+#    -p, --pods PODS                  List of pods to check
+#    -r, --restart COUNT              Threshold for number of restarts allowed
+#
+# NOTES:
+# => The filter used for the -f flag is in the form key=value. If multiple
+#    filters need to be specfied, use a comma. ex. foo=bar,red=color
+#
+# LICENSE:
+#   Barry Martin <nyxcharon@gmail.com>
+#   Released under the same terms as Sensu (the MIT license); see LICENSE
+#   for details.
+#
+
+require 'sensu-plugins-kubernetes/cli'
+
+class PodsRestarting < Sensu::Plugins::Kubernetes::CLI
+  @options = Sensu::Plugins::Kubernetes::CLI.options.dup
+
+  option :pod_list,
+         description: 'List of pods to check',
+         short: '-p PODS',
+         long: '--pods',
+         default: 'all'
+
+  option :restart_count,
+         description: 'Threshold for number of restarts allowed',
+         short: '-r COUNT',
+         long: '--restart',
+         proc: proc(&:to_i),
+         default: 10
+
+  option :pod_filter,
+         description: 'Selector filter for pods to be checked',
+         short: '-f FILTER',
+         long: '--filter'
+
+  option :exclude_namespace,
+         description: 'Exclude the specified list of namespaces',
+         short: '-n NAMESPACES',
+         long: '--exclude-namespace',
+         proc: proc { |a| a.split(',') },
+         default: ''
+
+  def run
+    pods_list = []
+    restarted_pods = []
+    pods = []
+    if config[:pod_filter].nil?
+      pods_list = parse_list(config[:pod_list])
+      pods = client.get_pods
+    else
+      pods = client.get_pods(label_selector: config[:pod_filter].to_s)
+      if pods.empty?
+        unknown 'The filter specified resulted in 0 pods'
+      end
+      pods_list = ['all']
+    end
+    pods.each do |pod|
+      next if pod.nil?
+      next if config[:exclude_namespace].include?(pod.metadata.namespace)
+      next unless pods_list.include?(pod.metadata.name) || pods_list.include?('all')
+      # Check restarts
+      next if pod.status.containerStatuses.nil?
+      pod.status.containerStatuses.each do |container|
+        if container.restartCount.to_i > config[:restart_count]
+          restarted_pods << "#{pod.metadata.namespace}.#{container.name}"
+        end
+      end
+    end
+
+    if restarted_pods.empty?
+      ok 'All pods are under restart threshold'
+    else
+      critical "Pods exceeded restart threshold: #{restarted_pods.join(' ')}"
+    end
+  rescue KubeException => e
+    critical 'API error: ' << e.message
+  end
+
+  def parse_list(list)
+    return list.split(',') if list && list.include?(',')
+    return [list] if list
+    ['']
+  end
+end

data/bin/check-kube-pods-running.rb

@@ -0,0 +1,104 @@
+#! /usr/bin/env ruby
+#
+#   check-kube-pods-running
+#
+# DESCRIPTION:
+# => Check if pods are in a running state
+#
+# OUTPUT:
+#   plain text
+#
+# PLATFORMS:
+#   Linux
+#
+# DEPENDENCIES:
+#   gem: sensu-plugin
+#   gem: kube-client
+#
+# Usage: ./check-kube-pods-running.rb (options)
+#         --ca-file CA-FILE            CA file to verify API server cert
+#         --cert CERT-FILE             Client cert to present
+#         --key KEY-FILE               Client key for the client cert
+#         --in-cluster                 Use service account authentication
+#         --password PASSWORD          If user is passed, also pass a password
+#     -s, --api-server URL             URL to API server
+#     -t, --token TOKEN                Bearer token for authorization
+#         --token-file TOKEN-FILE      File containing bearer token for authorization
+#     -u, --user USER                  User with access to API
+#     -v, --api-version VERSION        API version
+#     -n NAMESPACES,                   Exclude the specified list of namespaces
+#         --exclude-namespace
+#     -f, --filter FILTER              Selector filter for pods to be checked
+#     -p, --pods PODS                  List of pods to check
+# NOTES:
+# => The filter used for the -f flag is in the form key=value. If multiple
+#    filters need to be specfied, use a comma. ex. foo=bar,red=color
+#
+# LICENSE:
+#   Barry Martin <nyxcharon@gmail.com>
+#   Released under the same terms as Sensu (the MIT license); see LICENSE
+#   for details.
+#
+
+require 'sensu-plugins-kubernetes/cli'
+
+class AllPodsAreRunning < Sensu::Plugins::Kubernetes::CLI
+  @options = Sensu::Plugins::Kubernetes::CLI.options.dup
+
+  option :pod_list,
+         description: 'List of pods to check',
+         short: '-p PODS',
+         long: '--pods',
+         default: 'all'
+
+  option :pod_filter,
+         description: 'Selector filter for pods to be checked',
+         short: '-f FILTER',
+         long: '--filter'
+
+  option :exclude_namespace,
+         description: 'Exclude the specified list of namespaces',
+         short: '-n NAMESPACES',
+         long: '--exclude-namespace',
+         proc: proc { |a| a.split(',') },
+         default: ''
+
+  def run
+    pods_list = []
+    failed_pods = []
+    pods = []
+    if config[:pod_filter].nil?
+      pods_list = parse_list(config[:pod_list])
+      pods = client.get_pods
+    else
+      pods = client.get_pods(label_selector: config[:pod_filter].to_s)
+      if pods.empty?
+        unknown 'The filter specified resulted in 0 pods'
+      end
+      pods_list = ['all']
+    end
+    pods.each do |pod|
+      next if pod.nil?
+      next if config[:exclude_namespace].include?(pod.metadata.namespace)
+      next unless pods_list.include?(pod.metadata.name) || pods_list.include?('all')
+      next unless pod.status.phase != 'Succeeded' && !pod.status.conditions.nil?
+      if pod.status.conditions[1].status == 'False' # This is the Ready state
+        failed_pods << pod.metadata.name
+      end
+    end
+
+    if failed_pods.empty?
+      ok 'All pods are reporting as ready'
+    else
+      critical "Pods found in a non-ready state: #{failed_pods.join(' ')}"
+    end
+  rescue KubeException => e
+    critical 'API error: ' << e.message
+  end
+
+  def parse_list(list)
+    return list.split(',') if list && list.include?(',')
+    return [list] if list
+    ['']
+  end
+end

data/bin/check-kube-pods-runtime.rb

@@ -0,0 +1,116 @@
+#! /usr/bin/env ruby
+#
+#   check-kube-pods-runtime
+#
+# DESCRIPTION:
+# => Check if pods are running longer than expected
+#
+# OUTPUT:
+#   plain text
+#
+# PLATFORMS:
+#   Linux
+#
+# DEPENDENCIES:
+#   gem: sensu-plugin
+#   gem: kube-client
+#
+# USAGE:
+# -s, --api-server URL             URL to API server
+# -v, --api-version VERSION        API version. Defaults to 'v1'
+#     --in-cluster                 Use service account authentication
+#     --ca-file CA-FILE            CA file to verify API server cert
+#     --cert CERT-FILE             Client cert to present
+#     --key KEY-FILE               Client key for the client cert
+# -u, --user USER                  User with access to API
+#     --password PASSWORD          If user is passed, also pass a password
+#     --token TOKEN                Bearer token for authorization
+#     --token-file TOKEN-FILE      File containing bearer token for authorization
+# -c, --critical COUNT             Threshold for Pods to be critical
+# -f, --filter FILTER              Selector filter for pods to be checked
+# -p, --pods PODS                  List of pods to check
+# -w, --warn TIMEOUT               Threshold for pods to be in the pending state
+#
+# LICENSE:
+#   Barry Martin <nyxcharon@gmail.com>
+#   Released under the same terms as Sensu (the MIT license); see LICENSE
+#   for details.
+#
+
+require 'sensu-plugins-kubernetes/cli'
+
+class PodRuntime < Sensu::Plugins::Kubernetes::CLI
+  @options = Sensu::Plugins::Kubernetes::CLI.options.dup
+
+  option :pod_list,
+         description: 'List of pods to check',
+         short: '-p PODS',
+         long: '--pods',
+         default: 'all'
+
+  option :pod_filter,
+         description: 'Selector filter for pods to be checked',
+         short: '-f FILTER',
+         long: '--filter'
+
+  option :warn_timeout,
+         description: 'Threshold for pods to be in the pending state',
+         short: '-w TIMEOUT',
+         long: '--warn',
+         proc: proc(&:to_i)
+
+  option :critical_timeout,
+         description: 'Threshold for Pods to be critical',
+         short: '-c COUNT',
+         long: '--critical',
+         proc: proc(&:to_i)
+
+  def run
+    pods_list = []
+    pods = []
+    warn = false
+    crit = false
+    message = ''
+
+    if config[:pod_filter].nil?
+      pods_list = parse_list(config[:pod_list])
+      pods = client.get_pods
+    else
+      pods = client.get_pods(label_selector: config[:pod_filter].to_s)
+      pods_list = ['all']
+    end
+
+    pods.each do |pod|
+      next if pod.nil?
+      next unless pods_list.include?(pod.metadata.name) || pods_list.include?('all')
+      # Check for Running state
+      next unless pod.status.phase == 'Running'
+      pod_stamp = Time.parse(pod.status.startTime)
+      runtime = (Time.now.utc - pod_stamp.utc).to_i
+
+      if !config[:critical_timeout].nil? && runtime > config[:critical_timeout]
+        message << "#{pod.metadata.name} exceeds threshold #{config[:critical_timeout]} "
+        crit = true
+      elsif !config[:warn_timeout].nil? && runtime > config[:warn_timeout]
+        message << "#{pod.metadata.name} exceeds threshold #{config[:warn_timeout]} "
+        warn = true
+      end
+    end
+
+    if crit
+      critical message
+    elsif warn
+      warning message
+    else
+      ok 'All pods within threshold'
+    end
+  rescue KubeException => e
+    critical 'API error: ' << e.message
+  end
+
+  def parse_list(list)
+    return list.split(',') if list && list.include?(',')
+    return [list] if list
+    ['']
+  end
+end

data/bin/check-kube-service-available.rb

@@ -0,0 +1,121 @@
+#! /usr/bin/env ruby
+#
+#   check-kube-pods-service-available
+#
+# DESCRIPTION:
+# => Check if your kube services are up and ready
+#
+# OUTPUT:
+#   plain text
+#
+# PLATFORMS:
+#   Linux
+#
+# DEPENDENCIES:
+#   gem: sensu-plugin
+#   gem: kube-client
+#
+# USAGE:
+# -s, --api-server URL             URL to API server
+# -v, --api-version VERSION        API version. Defaults to 'v1'
+#     --in-cluster                 Use service account authentication
+#     --ca-file CA-FILE            CA file to verify API server cert
+#     --cert CERT-FILE             Client cert to present
+#     --key KEY-FILE               Client key for the client cert
+# -u, --user USER                  User with access to API
+#     --password PASSWORD          If user is passed, also pass a password
+#     --token TOKEN                Bearer token for authorization
+#     --token-file TOKEN-FILE      File containing bearer token for authorization
+# -l, --list SERVICES              List of services to check (required)
+# -p, --pending SECONDS            Time (in seconds) a pod may be pending for and be valid
+#
+# NOTES:
+#
+# LICENSE:
+#   Barry Martin <nyxcharon@gmail.com>
+#   Released under the same terms as Sensu (the MIT license); see LICENSE
+#   for details.
+#
+
+require 'sensu-plugins-kubernetes/cli'
+require 'time'
+
+class AllServicesUp < Sensu::Plugins::Kubernetes::CLI
+  @options = Sensu::Plugins::Kubernetes::CLI.options.dup
+
+  option :service_list,
+         description: 'List of services to check',
+         short: '-l SERVICES',
+         long: '--list',
+         required: true
+
+  option :pendingTime,
+         description: 'Time (in seconds) a pod may be pending for and be valid',
+         short: '-p SECONDS',
+         long: '--pending',
+         default: 0,
+         proc: proc(&:to_i)
+
+  def run
+    services = parse_list(config[:service_list])
+    failed_services = []
+    s = client.get_services
+    s.each do |a|
+      next unless services.include?(a.metadata.name)
+      # Build the selector key so we can fetch the corresponding pod
+      selector_key = []
+      services.delete(a.metadata.name)
+      a.spec.selector.to_h.each do |k, v|
+        selector_key << "#{k}=#{v}"
+      end
+      # Get the pod
+      pod = nil
+      begin
+        pod = client.get_pods(label_selector: selector_key.join(',').to_s)
+      rescue
+        failed_services << a.metadata.name.to_s
+      end
+      # Make sure our pod is running
+      next if pod.nil?
+      pod_available = false
+      pod.each do |p|
+        case p.status.phase
+        when 'Pending'
+          next if p.status.startTime.nil?
+          if (Time.now - Time.parse(p.status.startTime)).to_i < config[:pendingTime]
+            pod_available = true
+            break
+          end
+        when 'Running'
+          p.status.conditions.each do |c|
+            next unless c.type == 'Ready'
+            if c.status == 'True'
+              pod_available = true
+              break
+            end
+            break if pod_available
+          end
+        end
+        failed_services << "#{p.metadata.namespace}.#{p.metadata.name}" if pod_available == false
+      end
+    end
+
+    if failed_services.empty? && services.empty?
+      ok 'All services are reporting as up'
+    end
+
+    if !failed_services.empty?
+      critical "All services are not ready: #{failed_services.join(' ')}"
+    else
+      critical "Some services could not be checked: #{services.join(' ')}"
+    end
+  rescue KubeException => e
+    critical 'API error: ' << e.message
+  end
+
+  def parse_list(list)
+    return list.split(',') if list && list.include?(',')
+    return [list] if list
+    ['']
+  end
+end