riemann-tools 1.0.0 → 1.2.0

data/tools/riemann-rabbitmq/lib/riemann/tools/rabbitmq.rb

@@ -0,0 +1,269 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+module Riemann
+  module Tools
+    class Rabbitmq
+      include Riemann::Tools
+
+      require 'faraday'
+      require 'json'
+      require 'uri'
+
+      opt :read_timeout, 'Faraday read timeout', type: :int, default: 2
+      opt :open_timeout, 'Faraday open timeout', type: :int, default: 1
+
+      opt :monitor_user, 'RabbitMQ monitoring user', type: :string
+      opt :monitor_pass, 'RabbitMQ monitoring user password', type: :string
+      opt :monitor_port, 'RabbitMQ monitoring port', type: :int, default: 15_672
+      opt :monitor_host, 'RabbitMQ monitoring host', type: :string, default: 'localhost'
+      opt :monitor_use_tls, 'RabbitMQ use tls', type: :bool, default: false
+
+      opt :max_queue_size, 'max number of items in a queue that is acceptable', type: :int, default: 1_000_000
+      opt :ignore_max_size_queues, "A regular expression to match queues that shouldn't be size-checked", type: :string
+
+      opt :node, 'Specify a node to monitor', type: :strings
+
+      def base_url
+        protocol = 'http'
+        protocol = 'https' if options[:monitor_use_tls] && (options[:monitor_use_tls] == true)
+        "#{protocol}://#{options[:monitor_user]}:#{options[:monitor_pass]}@#{options[:monitor_host]}:#{options[:monitor_port]}/api"
+      end
+
+      def overview_url
+        "#{base_url}/overview"
+      end
+
+      def node_url(node)
+        "#{base_url}/nodes/#{node}"
+      end
+
+      def queues_url
+        "#{base_url}/queues"
+      end
+
+      def event_host
+        options[:event_host] || :monitor_host
+      end
+
+      def safe_get(uri, event_host)
+        # Handle connection timeouts
+        response = nil
+        begin
+          connection = Faraday.new(uri)
+          response = connection.get do |req|
+            req.options[:timeout] = options[:read_timeout]
+            req.options[:open_timeout] = options[:open_timeout]
+          end
+          report(
+            host: event_host,
+            service: 'rabbitmq monitoring',
+            state: 'ok',
+            description: 'Monitoring operational',
+          )
+        rescue StandardError => e
+          report(
+            host: event_host,
+            service: 'rabbitmq monitoring',
+            state: 'critical',
+            description: "HTTP connection error: #{e.class} - #{e.message}",
+          )
+        end
+        response
+      end
+
+      def check_queues
+        response = safe_get(queues_url, event_host)
+        max_size_check_filter = (Regexp.new(options[:ignore_max_size_queues]) if options[:ignore_max_size_queues])
+
+        return if response.nil?
+
+        if response.status != 200
+          report(
+            host: event_host,
+            service: 'rabbitmq.queue',
+            state: 'critical',
+            description: "HTTP connection error to /api/queues: #{response.status} - #{response.body}",
+          )
+        else
+          report(
+            host: event_host,
+            service: 'rabbitmq.queue',
+            state: 'ok',
+            description: 'HTTP connection ok',
+          )
+
+          json = JSON.parse(response.body)
+
+          json.each do |queue|
+            svc = "rabbitmq.queue.#{queue['vhost']}.#{queue['name']}"
+            errs = []
+
+            errs << 'Queue has jobs but no consumers' if !queue['messages_ready'].nil? && (queue['messages_ready']).positive? && (queue['consumers']).zero?
+
+            errs << "Queue has #{queue['messages_ready']} jobs" if (max_size_check_filter.nil? || queue['name'] !~ (max_size_check_filter)) && !queue['messages_ready'].nil? && (queue['messages_ready'] > options[:max_queue_size])
+
+            if errs.empty?
+              report(
+                host: event_host,
+                service: svc,
+                state: 'ok',
+                description: 'Queue is looking good',
+              )
+            else
+              report(
+                host: event_host,
+                service: svc,
+                state: 'critical',
+                description: errs.join('; '),
+              )
+            end
+
+            stats = (queue['message_stats'] || {}).merge(
+              'messages'                        => queue['messages'],
+              'messages_details'                => queue['messages_details'],
+              'messages_ready'                  => queue['messages_ready'],
+              'messages_ready_details'          => queue['messages_ready_details'],
+              'messages_unacknowledged'         => queue['messages_unacknowledged'],
+              'messages_unacknowledged_details' => queue['messages_unacknowledged_details'],
+              'consumers'                       => queue['consumers'],
+              'memory'                          => queue['memory'],
+            )
+
+            stats.each_pair do |k, v|
+              service = "#{svc}.#{k}"
+              metric = if k =~ (/details$/) && !v.nil?
+                         v['rate']
+                       else
+                         v
+                       end
+
+              # TODO: Set state via thresholds which can be configured
+
+              report(
+                host: event_host,
+                service: service,
+                metric: metric,
+                description: 'RabbitMQ monitor',
+              )
+            end
+          end
+        end
+      end
+
+      def check_overview
+        uri = URI(overview_url)
+        response = safe_get(uri, event_host)
+
+        return if response.nil?
+
+        json = JSON.parse(response.body)
+
+        if response.status != 200
+          report(
+            host: event_host,
+            service: 'rabbitmq',
+            state: 'critical',
+            description: "HTTP connection error: #{response.status} - #{response.body}",
+          )
+        else
+          report(
+            host: event_host,
+            service: 'rabbitmq monitoring',
+            state: 'ok',
+            description: 'HTTP connection ok',
+          )
+
+          %w[message_stats queue_totals object_totals].each do |stat|
+            # NOTE: / BUG ?
+            # Brand new servers can have blank message stats. Is this ok?
+            # I can't decide.
+            next if json[stat].empty?
+
+            json[stat].each_pair do |k, v|
+              service = "rabbitmq.#{stat}.#{k}"
+              metric = if k =~ /details$/
+                         v['rate']
+                       else
+                         v
+                       end
+
+              # TODO: Set state via thresholds which can be configured
+
+              report(
+                host: event_host,
+                service: service,
+                metric: metric,
+                description: 'RabbitMQ monitor',
+              )
+            end
+          end
+        end
+      end
+
+      def check_node
+        opts[:node].each do |n|
+          uri = URI(node_url(n))
+          response = safe_get(uri, event_host)
+
+          break if response.nil?
+
+          if response.status != 200
+            if response.status == 404
+              report(
+                host: event_host,
+                service: "rabbitmq.node.#{n}",
+                state: 'critical',
+                description: 'Node was not found in the cluster',
+              )
+            else
+              report(
+                host: event_host,
+                service: "rabbitmq.node.#{n}",
+                state: 'critical',
+                description: "HTTP error: #{response.status} - #{response.body}",
+              )
+            end
+            break
+          end
+
+          json = JSON.parse(response.body)
+
+          if json['mem_alarm']
+            report(
+              host: event_host,
+              service: "rabbitmq.node.#{n}",
+              state: 'critical',
+              description: 'Memory alarm has triggered; job submission throttled',
+            )
+            break
+          end
+
+          if json['disk_free_alarm']
+            report(
+              host: event_host,
+              service: "rabbitmq.node.#{n}",
+              state: 'critical',
+              description: 'Disk free alarm has triggered; job submission throttled',
+            )
+            break
+          end
+
+          report(
+            host: event_host,
+            service: "rabbitmq.node.#{n}",
+            state: 'ok',
+            description: 'Node looks OK to me',
+          )
+        end
+      end
+
+      def tick
+        check_overview
+        check_node if opts[:node]
+        check_queues
+      end
+    end
+  end
+end

data/tools/riemann-riak/{Rakefile.rb → Rakefile}

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rubygems'
 require 'rubygems/package_task'
 require 'rdoc/task'
@@ -9,10 +11,8 @@ ENV['COPYFILE_DISABLE'] = 'true'
 
 # Gemspec
 gemspec = Gem::Specification.new do |s|
-  s.rubyforge_project = 'riemann-riak'
-
   s.name = 'riemann-riak'
-  s.version = '0.1.2'
+  s.version = '0.1.4'
   s.author = 'Kyle Kingsbury'
   s.email = 'aphyr@aphyr.com'
   s.homepage = 'https://github.com/riemann/riemann-tools'
@@ -20,14 +20,13 @@ gemspec = Gem::Specification.new do |s|
   s.summary = 'Submits riak stats to riemann.'
   s.license = 'MIT'
 
-  s.add_dependency 'riemann-tools', '>= 0.2.13'
-  s.add_dependency 'yajl-ruby', '>= 1.1.0'
+  s.add_runtime_dependency 'riemann-tools', '~> 1.0', '>= 1.1.1'
+  s.add_runtime_dependency 'yajl-ruby', '~> 1.4', '>= 1.4.3'
 
-  s.files = FileList['bin/*', 'LICENSE', 'README.md'].to_a
+  s.files = FileList['bin/*', 'lib/**/*.rb', 'LICENSE', 'README.md'].to_a
   s.executables |= Dir.entries('bin/')
-  s.has_rdoc = false
 
-  s.required_ruby_version = '>= 1.8.7'
+  s.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
 end
 
 Gem::PackageTask.new gemspec do |p|

data/tools/riemann-riak/bin/riemann-riak

@@ -1,331 +1,8 @@
 #!/usr/bin/env ruby
-Process.setproctitle($0)
+# frozen_string_literal: true
 
-# Forwards information on a Riak node to Riemann.
+Process.setproctitle($PROGRAM_NAME)
 
-require 'riemann/tools'
-
-class Riemann::Tools::Riak
-  include Riemann::Tools
-  require 'net/http'
-  require 'net/https'
-  require 'yajl/json_gem'
-
-  opt :riak_host, "Riak host for stats <IP> or SSL http(s)://<IP>", :default => Socket.gethostname
-  opt :data_dir, "Riak data directory", :default => '/var/lib/riak'
-  opt :stats_port, "Riak HTTP port for stats", :default => 8098
-  opt :stats_path, "Riak HTTP stats path", :default => '/stats'
-  opt :node_name, "Riak erlang node name", :default => "riak@#{Socket.gethostname}"
-  opt :cookie, "Riak cookie to use", :default => "riak"
-
-  opt :get_50_warning, "FSM 50% get time warning threshold (ms)", :default => 1000
-  opt :put_50_warning, "FSM 50% put time warning threshold (ms)", :default => 1000
-  opt :get_95_warning, "FSM 95% get time warning threshold (ms)", :default => 2000
-  opt :put_95_warning, "FSM 95% put time warning threshold (ms)", :default => 2000
-  opt :get_99_warning, "FSM 99% get time warning threshold (ms)", :default => 10000
-  opt :put_99_warning, "FSM 99% put time warning threshold (ms)", :default => 10000
-
-  def initialize
-    detect_features
-
-    @httpstatus = true
-
-    begin
-      uri = URI.parse(opts[:riak_host])
-      if uri.host == nil
-        uri.host = opts[:riak_host]
-      end
-      http = Net::HTTP.new(uri.host, opts[:stats_port])
-      http.use_ssl = uri.scheme == 'https'
-      if http.use_ssl?
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      end
-      http.start do |h|
-        h.get opts[:stats_path]
-      end
-    rescue => _e
-      @httpstatus = false
-    end
-
-    # we're going to override the emulator setting to allow users to
-    # dynamically input the cookie
-    # this is done only once - hopefully it doesn't get overridden.
-    ENV['ERL_AFLAGS'] = "-setcookie #{opts[:cookie]}"
-  end
-
-  # Identifies whether escript and riak-admin are installed
-  def detect_features
-    @escript = true # Whether escript is present on this machine
-    @riakadmin = true # Whether riak-admin is present
-
-    if `which escript` =~ /^\s*$/
-      @escript = false
-    end
-
-    if `which riak-admin` =~ /^\s*$/
-      @riakadmin = false
-    end
-  end
-
-  def check_ring
-    str = if @escript
-      str = `#{File.expand_path(File.dirname(__FILE__))}/riemann-riak-ring #{opts[:node_name]}`.chomp
-    elsif @riakadmin
-      str = `riak-admin ringready`
-    else
-      nil
-    end
-
-    return if str.nil?
-
-    if str =~ /^TRUE/
-      report(
-        :host => opts[:riak_host],
-        :service => 'riak ring',
-        :state => 'ok',
-        :description => str
-      )
-    else
-      report(
-        :host => opts[:riak_host],
-        :service => 'riak ring',
-        :state => 'warning',
-        :description => str
-      )
-    end
-  end
-
-  def check_keys
-    keys = `#{File.expand_path(File.dirname(__FILE__))}/riemann-riak-keys #{opts[:node_name]}`.chomp
-    if keys =~ /^\d+$/
-      report(
-        :host => opts[:riak_host],
-        :service => 'riak keys',
-        :state => 'ok',
-        :metric => keys.to_i,
-        :description => keys
-      )
-    else
-      report(
-        :host => opts[:riak_host],
-        :service => 'riak keys',
-        :state => 'unknown',
-        :description => keys
-      )
-    end
-  end
-
-  def check_transfers
-    str = if @riakadmin
-      `riak-admin transfers`
-    else
-      nil
-    end
-
-    return if str.nil?
-
-    if str =~ /'#{opts[:node_name]}' waiting to handoff (\d+) partitions/
-      report(
-        :host => opts[:riak_host],
-        :service => 'riak transfers',
-        :state   => 'critical',
-        :metric  => $1.to_i,
-        :description => "waiting to handoff #{$1} partitions"
-      )
-    else
-      report(
-        :host => opts[:riak_host],
-        :service => 'riak transfers',
-        :state   => 'ok',
-        :metric  => 0,
-        :description => "No pending transfers"
-      )
-    end
-  end
-
-  def check_disk
-    gb = `du -Ls #{opts[:data_dir]}`.split(/\s+/).first.to_i / (1024.0**2)
-    report(
-      :host => opts[:riak_host],
-      :service => 'riak disk',
-      :state => 'ok',
-      :metric => gb,
-      :description => "#{gb} GB in #{opts[:data_dir]}"
-    )
-  end
-
-  # Returns the riak stat for the given fsm type and percentile.
-  def fsm_stat(type, property, percentile)
-    "node_#{type}_fsm_#{property}_#{percentile == 50 ? 'median' : percentile}"
-  end
-
-  # Returns the alerts state for the given fsm.
-  def fsm_state(type, percentile, val)
-    limit = opts["#{type}_#{percentile}_warning".to_sym]
-    case val
-    when 0 .. limit
-      'ok'
-    when limit .. limit * 2
-      'warning'
-    else
-      'critical'
-    end
-  end
-
-  # Get current stats via HTTP
-  def stats_http
-    begin
-      uri = URI.parse(opts[:riak_host])
-      if uri.host == nil
-        uri.host = opts[:riak_host]
-      end
-      http = Net::HTTP.new(uri.host, opts[:stats_port])
-      http.use_ssl = uri.scheme == 'https'
-      if http.use_ssl?
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      end
-      res = http.start do |h|
-        h.get opts[:stats_path]
-      end
-    rescue => e
-      report(
-        :host => opts[:riak_host],
-        :service => 'riak',
-        :state => 'critical',
-        :description => "error fetching #{opts[:riak_host]}:#{opts[:stats_port]} #{e.class}, #{e.message}"
-      )
-      raise
-    end
-
-    if res.code.to_i == 200
-      return JSON.parse(res.body)
-    else
-      report(
-        :host => opts[:riak_host],
-        :service => 'riak',
-        :state => 'critical',
-        :description => "stats returned HTTP #{res.code}:\n\n#{res.body}"
-      )
-      raise "Can't fetch stats via HTTP: #{res.core}:\n\n#{res.body}"
-    end
-  end
-
-  # Get current stats via riak-admin
-  def stats_riak_admin
-    str = `riak-admin status`
-    raise "riak-admin failed" unless $? == 0
-    Hash[str.split(/\n/).map{|i| i.split(/ : /)}]
-  end
-
-  # Get current stats as a hash
-  def stats
-    if @httpstatus
-      stats_http
-    elsif @riakadmin
-      stats_riak_admin
-    else
-      report(
-        :host => opts[:riak_host],
-        :service => 'riak',
-        :state => 'critical',
-        :description => "No mechanism for fetching Riak stats: neither HTTP nor riak-admin available."
-      )
-      raise "No mechanism for fetching Riak stats: neither HTTP nor riak-admin available."
-    end
-  end
-
-  def core_services
-    ['vnode_gets',
-     'vnode_puts',
-     'node_gets',
-     'node_puts',
-     'node_gets_set',
-     'node_puts_set',
-     'read_repairs']
-  end
-
-  def fsm_types
-    [{'get' => 'time'}, {'put' => 'time'},
-     {'get' => 'set_objsize'}]
-  end
-
-  def fsm_percentiles
-    [50, 95, 99]
-  end
-
-  # Reports current stats to Riemann
-  def check_stats
-    begin
-      stats = self.stats
-    rescue => e
-      event = {:state => 'critical',
-               :description => e.message,
-               :host => opts[:riak_host]}
-      # Report errors
-      report(event.merge(:service => 'riak'))
-      core_services.each do |s|
-        report(event.merge(:service => "riak #{s}"))
-      end
-      fsm_types.each do |typespec|
-        typespec.each do |type, prop|
-          fsm_percentiles.each do |percentile|
-            report(event.merge(:service => "riak #{type} #{prop} #{percentile}"))
-          end
-        end
-      end
-      return
-    end
-
-    # Riak itself
-    report(
-      :host => opts[:riak_host],
-      :service => 'riak',
-      :state => 'ok'
-    )
-
-    # Gets/puts/rr
-    core_services.each do |s|
-      report(
-        :host => opts[:riak_host],
-        :service => "riak #{s}",
-        :state => 'ok',
-        :metric => stats[s].to_i/60.0,
-        :description => "#{stats[s].to_i/60.0}/sec"
-      )
-    end
-
-    # FSMs
-    fsm_types.each do |typespec|
-      typespec.each do |type, prop|
-        fsm_percentiles.each do |percentile|
-          val = stats[fsm_stat(type, prop, percentile)].to_i || 0
-          val = 0 if val == 'undefined'
-          val /= 1000.0 if prop == 'time' # Convert us to ms
-          if prop == 'time'
-            state = fsm_state(type,  percentile, val)
-          else
-            state = "ok"
-          end
-          report(
-            :host => opts[:riak_host],
-            :service => "riak #{type} #{prop} #{percentile}",
-            :state => state,
-            :metric => val,
-            :description => "#{val} ms"
-          )
-        end
-      end
-    end
-  end
-
-  def tick
-    # This can utterly destroy a cluster, so we disable
-    # check_keys
-    check_stats
-    check_ring
-    check_disk
-    check_transfers
-  end
-end
+require 'riemann/tools/riak'
 
 Riemann::Tools::Riak.run

data/tools/riemann-riak/bin/riemann-riak-keys

@@ -1,5 +1,4 @@
 #!/usr/bin/env escript
-Process.setproctitle($0)
 %%! -name riakstatuscheck@127.0.0.1 -hidden
 
 main([]) -> main(["riak@127.0.0.1"]);

data/tools/riemann-riak/bin/riemann-riak-ring

@@ -1,5 +1,4 @@
 #!/usr/bin/env escript
-Process.setproctitle($0)
 %%! -name riakstatuscheck@127.0.0.1 -hidden
 
 main([]) -> main(["riak@127.0.0.1"]);