sensu-plugins-prometheus-checks 2.3.0

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/check_prometheus.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'sensu/plugins/prometheus/checks/runner'
+
+config_file = ARGV[0] || 'config.yml'
+abort("Can't find configuration file at '#{config_file}'") \
+  unless File.exist?(config_file)
+
+runner = nil
+
+begin
+  runner = Sensu::Plugins::Prometheus::Checks::Runner.new(
+    YAML.load_file(config_file)
+  )
+  runner.run
+rescue RuntimeError => e
+  puts "ERROR: #{e}"
+  exit(1)
+end
+
+puts("\n")
+puts(runner.output)
+exit(runner.status)

data/docker-compose.yml

@@ -0,0 +1,53 @@
+version: '2'
+
+networks:
+  default:
+    external:
+      name: check_prometheus
+
+services:
+  promtheues:
+    image: quay.io/prometheus/prometheus:v1.0.1
+    container_name: prometheus
+    volumes:
+      - ./spec/docker/prometheus/:/etc/prometheus/
+    command:
+      - '-config.file=/etc/prometheus/prometheus.yml'
+      - '-storage.local.path=/prometheus'
+    expose:
+      - 19090
+    ports:
+      - 19090:9090
+  node-exporter1:
+    image: prom/node-exporter:0.12.0
+    expose:
+      - 19100
+    ports:
+      - 19100:9100
+    volumes:
+      - ./spec/docker/node-exporter/node1/:/etc/node-exporter/
+    command:
+      - "-collectors.enabled=textfile"
+      - "-collector.textfile.directory=/etc/node-exporter/"
+  node-exporter2:
+    image: prom/node-exporter:0.12.0
+    expose:
+      - 19101
+    ports:
+      - 19101:9100
+    volumes:
+      - ./spec/docker/node-exporter/node2/:/etc/node-exporter/
+    command:
+      - "-collectors.enabled=textfile"
+      - "-collector.textfile.directory=/etc/node-exporter/"
+  node-exporter3:
+    image: prom/node-exporter:0.12.0
+    expose:
+      - 19102
+    ports:
+      - 19102:9100
+    volumes:
+      - ./spec/docker/node-exporter/node3/:/etc/node-exporter/
+    command:
+      - "-collectors.enabled=textfile"
+      - "-collector.textfile.directory=/etc/node-exporter/"

data/lib/sensu/plugins/events/dispatcher.rb

@@ -0,0 +1,52 @@
+require 'json'
+require 'socket'
+require 'sensu/plugins/utils/log'
+
+module Sensu
+  module Plugins
+    module Events
+      # Helper class to dispatch events into Sensu socket.
+      class Dispatcher
+        include Sensu::Plugins::Utils::Log
+
+        def initialize
+          @sensu_address = '127.0.0.1'
+          @sensu_port = 3030
+          read_env_address_and_port
+
+          log.debug("Sensu at '#{@sensu_address}':'#{@sensu_port}'")
+        end
+
+        # Send accumulated events into Sensu's socket, unless environment
+        # variable PROM_DEBUG is set.
+        def dispatch(event)
+          if ENV.key?('PROM_DEBUG')
+            log.debug("PROM_DEBUG set, not dispatching event to Sensu: #{event}")
+            return
+          end
+
+          # :nocov:
+          begin
+            s = TCPSocket.open(@sensu_address, @sensu_port)
+            s.puts(JSON.generate(event))
+            s.close
+          rescue SystemCallError => e
+            log.error("Sensu is refusing connections! Error: '#{e}'")
+            raise("Sensu is not avilable at '#{@sensu_address}:#{@sensu_port}'")
+          end
+          # :nocov:
+        end
+
+        private
+
+        # Read Sensu address and port from environment.
+        def read_env_address_and_port
+          @sensu_address = ENV['SENSU_SOCKET_ADDRESS'] \
+            if ENV.key?('SENSU_SOCKET_ADDRESS')
+          @sensu_port = ENV['SENSU_SOCKET_PORT'].to_i \
+            if ENV.key?('SENSU_SOCKET_PORT')
+        end
+      end
+    end
+  end
+end

data/lib/sensu/plugins/prometheus/checks.rb

@@ -0,0 +1,49 @@
+module Sensu
+  module Plugins
+    module Prometheus
+      # Static methods to help on a given check evaluation. This module is
+      # designed to be included where the check evaluation will happen.
+      module Checks
+        # Given current result, warning and critical levels, it will return a
+        # integer with the current level, zero is success.
+        def evaluate(result, warn, crit)
+          result = result.to_f
+          warn = warn.to_f
+          crit = crit.to_f
+
+          status = 3
+          if result < warn
+            status = 0
+          elsif result >= crit
+            status = 2
+          elsif result >= warn
+            status = 1
+          end
+
+          status
+        end
+
+        # Return zero if result and value are the same.
+        def equals(result, value)
+          status = 2
+          status = 0 if result.to_f == value.to_f
+          status
+        end
+
+        # Return zero if result is below value.
+        def below(result, value)
+          status = 2
+          status = 0 if result.to_f < value.to_f
+          status
+        end
+
+        # Return zero if result is above value.
+        def above(result, value)
+          status = 2
+          status = 0 if result.to_f > value.to_f
+          status
+        end
+      end
+    end
+  end
+end

data/lib/sensu/plugins/prometheus/checks/namespace.rb

@@ -0,0 +1,21 @@
+module Sensu
+  module Plugins
+    module Prometheus
+      module Checks
+        # Helper to transform a hash into local methods.
+        class Namespace
+          def initialize(hash)
+            hash.each do |key, value|
+              singleton_class.send(:define_method, key) { value }
+            end
+          end
+
+          # Wrap around `binding` method.
+          def namespace_binding
+            binding
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sensu/plugins/prometheus/checks/output.rb

@@ -0,0 +1,47 @@
+require 'erb'
+require 'ostruct'
+
+require 'sensu/plugins/prometheus/checks/namespace'
+
+module Sensu
+  module Plugins
+    module Prometheus
+      module Checks
+        # Handles the final output of checks, creates a custom message based on
+        # template and informed variables.
+        class Output
+          TEMPLATES = {
+            'disk' => \
+              "Disk: <%= value %>%, Mountpoint: <%= cfg['mount'] %> |disk=<%= value %>",
+            'disk_all' => \
+              "Disk: <%= cfg['mount'] %>, Inode Usage: <%= value %>% |inodes=<%= value %>",
+            'load_per_cpu' => \
+              'Load: <%= value %>|load=<%= value %>',
+            'load_per_cluster' => \
+              'Cluster Load: <%= value %> |load=<%= value %>',
+            'load_per_cluster_minus_n' => \
+              'Cluster Load: <%= value %> |load=<%= value %>',
+            'inode' => \
+              "Disk: <%= cfg['mount'] %>, Inodes: <%= value %>% |inodes=<%= value %>",
+            'memory' => \
+              'Memory <%= value %>% |memory=<%= value %>',
+            'memory_per_cluster' => \
+              'Cluster Memory: <%= value %>% |memory=<%= value %>"',
+            'predict_disk_all' => \
+              "Disks to run out of space in the next <%= cfg['days'] %> days: <%= value %>",
+            'service' => \
+              "Service: <%= cfg['name'] %> (<%= cfg['state'] %>=<%= value %>)"
+          }.freeze
+
+          def render(template_name, vars)
+            template_name = template_name.to_s
+            raise "Can't find template for '#{template_name}'" \
+              if !TEMPLATES.key?(template_name) || TEMPLATES[template_name].empty?
+            ns = Sensu::Plugins::Prometheus::Checks::Namespace.new(vars)
+            ERB.new(TEMPLATES[template_name]).result(ns.namespace_binding)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sensu/plugins/prometheus/checks/runner.rb

@@ -0,0 +1,267 @@
+require 'sensu/plugins/utils/log'
+require 'sensu/plugins/events/dispatcher'
+require 'sensu/plugins/prometheus/client'
+require 'sensu/plugins/prometheus/metrics'
+require 'sensu/plugins/prometheus/checks'
+require 'sensu/plugins/prometheus/checks/output'
+
+module Sensu
+  module Plugins
+    module Prometheus
+      module Checks
+        # Execute the configured checks, evaluate the results and set a final
+        # output and status.
+        class Runner
+          include Sensu::Plugins::Utils::Log
+          include Sensu::Plugins::Prometheus::Checks
+
+          attr_reader :status, :output, :events
+
+          # Does basic configuration validation and start the object the methods
+          # on this class will consume.
+          def initialize(config)
+            raise 'Configuration is empty, abort!' \
+              if config.nil? || config.empty?
+            raise "Configuration does not specify 'config' section!" \
+              unless config.key?('config')
+
+            config['checks'] = [] unless config.key?('checks')
+            config['custom'] = [] unless config.key?('custom')
+
+            @config = config
+            @events = []
+            @status = 0
+            @output = ''
+            @source_nodename_map = nil
+
+            @prometheus = Sensu::Plugins::Prometheus::Client.new
+            @metrics =  Sensu::Plugins::Prometheus::Metrics.new(@prometheus)
+            @tmpl = Sensu::Plugins::Prometheus::Checks::Output.new
+            @dispatcher = Sensu::Plugins::Events::Dispatcher.new
+          end
+
+          # Drives the evaluation of regular and custom checks, then calls for
+          # the final analysis on collected events.
+          def run
+            evaluate_checks if @config.key?('checks')
+            evaluate_custom if @config.key?('custom')
+            evaluate_and_dispatch_events
+          end
+
+          private
+
+          # Wrap around Metrics object and capture exception.
+          def collect_metrics(name, check_cfg)
+            metrics = []
+            begin
+              # invoke method name method on metric object
+              metrics = @metrics.send(name, check_cfg)
+            # :nocov:
+            rescue NoMethodError => e
+              log.error(
+                "Method '#{name}' is not present on Metrics object: '#{e}'"
+              )
+            end
+            # :nocov:
+            metrics
+          end
+
+          # Apply evaluation to all pre-defined checks.
+          def evaluate_checks
+            log.info("Evaluating Checks: '#{@config['checks'].length}'")
+
+            @config['checks'].each do |check|
+              check_name = check['check']
+              check_cfg = check['cfg']
+
+              collect_metrics(check_name, check_cfg).each do |metric|
+                status = 0
+
+                # on service it will come with "state_required" flag
+                if check_name == 'service'
+                  # adding defaults in case they are not set
+                  check_cfg = check_cfg.merge(
+                    'state' => 'active',
+                    'state_required' => 1
+                  )
+                  # giving a service hint by adding it's name
+                  check_name = "service_#{check_cfg['name']}"
+                  status = equals(metric['value'], check_cfg['state_required'])
+                else
+                  # normal threshold evaluation
+                  status = evaluate(
+                    metric['value'],
+                    check_cfg['warn'],
+                    check_cfg['crit']
+                  )
+                end
+
+                template_variables = metric
+                template_variables['cfg'] = check_cfg
+
+                append_event(
+                  "check_#{check_name}",
+                  @tmpl.render(check['check'], template_variables),
+                  status,
+                  metric['source']
+                )
+              end
+            end
+          end
+
+          # Apply checks based on custom Prometheus queries and custom check
+          # config.
+          def evaluate_custom
+            log.info("Evaluating Custom: '#{@config['custom'].length}'")
+
+            @config['custom'].each do |custom|
+              # invoke "custom" method in metrics object
+              collect_metrics('custom', custom).each do |metric|
+                value = metric['value']
+                name = custom['name']
+
+                if custom.key?('check') && !custom['check']['type'].empty?
+                  # calling local method to determine metric status
+                  status = send(
+                    custom['check']['type'],
+                    value,
+                    custom['check']['value']
+                  )
+                elsif custom.key?('cfg')
+                  # normal threshold evaluation
+                  status = evaluate(
+                    value,
+                    custom['cfg']['warn'],
+                    custom['cfg']['crit']
+                  )
+                else
+                  log.warn(
+                    "Custom check does not have 'check' or 'cfg', can't be evaluated"
+                  )
+                  status = 3
+                end
+
+                log.debug("Custom Check: name='#{name}', value='#{value}'")
+
+                # making sure the custom check has the status defined
+                output = if custom['msg'].key?(status)
+                           custom['msg'][status].to_s
+                         else
+                           'No output message defined for this check'
+                         end
+
+                append_event(
+                  "custom_#{name}",
+                  output,
+                  status,
+                  metric['source']
+                )
+              end
+            end
+          end
+
+          # Classify and select whitelisted events to dispatch. Also prepares
+          # the final status and output message.
+          def evaluate_and_dispatch_events
+            non_successful_events = []
+
+            @events.reverse_each do |event|
+              # skipping events that are not whitelisted
+              if @config['config'].key?('whitelist') && event['source'] !~ /#{@config['config']['whitelist']}/
+                @events.delete(event)
+                log.debug(
+                  "Skipping event! Source '#{event['source']}' does not " \
+                    "match /#{@config['config']['whitelist']}/"
+                )
+                next
+              end
+              # removing source key to use local's sensu source name (hostname)
+              if @config.key?('config') && \
+                 @config['config'].key?('use_default_source') && \
+                 @config['config']['use_default_source']
+                log.debug("Removing 'source' from event, using Sensu's default")
+                event.delete('source')
+              end
+              # selecting the non-succesful events
+              non_successful_events << event if event['status'] != 0
+              # dispatching event to Sensu
+              @dispatcher.dispatch(event)
+            end
+
+            # setting up final status and output message
+            amount_checks = @config['checks'].length + @config['custom'].length
+            amount_events = @events.length
+
+            if non_successful_events.empty?
+              @status = 0
+              @output = \
+                "OK: Ran #{amount_checks} checks succesfully on #{amount_events} events!"
+            else
+              log.debug("#{non_successful_events.length} failed events")
+              @status = 1
+              non_successful_events.sort_by { |e| e['status'] } .reverse.each do |event|
+                @output << ' | ' unless @output.empty?
+                @output << "Source: #{event['source']}, " \
+                  "Check: #{event['name']}, " \
+                  "Output: #{event['output']}, " \
+                  "Status: #{event['status']}"
+              end
+            end
+
+            log.debug("Ran #{amount_checks}, and collected #{amount_events} events")
+            log.debug("Final Status: #{@status}")
+            log.debug("Final Output: #{@output}")
+          end
+
+          # Query Prometheus to discover the nodenames per instance, found on
+          # the last day, and sanitize query events into a hash, returned by
+          # this method.
+          def source_nodename_map
+            map = {}
+            @prometheus.query('max_over_time(node_uname_info[1d])').each do |result|
+              source = result['metric']['instance']
+              nodename = result['metric']['nodename'].split('.', 2)[0]
+              log.info("[node_exporter] instance: '#{source}', nodename: '#{nodename}'")
+              map[source] = nodename
+            end
+            log.warn('Unable to query the node_exporter intances from Prometheus') \
+              if map.empty?
+            map
+          end
+
+          # Remove chars that are not allowed in Sensu.
+          def sensu_safe(string)
+            string.gsub(/[^\w\.-]+/, '_')
+          end
+
+          # Append an event on the pool, making string safe for Sensu, checking
+          # "source" against "source_nodename_map" and composing "address" using
+          # configuration "domain" entry.
+          def append_event(name, output, status, source)
+            log.info(
+              "[#{status}] check: '#{name}', output: '#{output}', source: '#{source}'"
+            )
+
+            # let source-nodename mapping avialable
+            @source_nodename_map = source_nodename_map \
+              if @source_nodename_map.nil?
+
+            # translating node_exporter hostname into nodename plus domain
+            nodename = @source_nodename_map[source] || source
+            address = "#{nodename}.#{@config['config']['domain']}"
+
+            @events << {
+              'address' => sensu_safe(address),
+              'name' => sensu_safe(name),
+              'occurrences' => @config['config']['occurrences'] || 1,
+              'output' => output,
+              'reported_by' => @config['config']['reported_by'],
+              'status' => status,
+              'source' => sensu_safe(nodename)
+            }
+          end
+        end
+      end
+    end
+  end
+end