riemann-tools 1.1.1 → 1.4.0

data/tools/riemann-rabbitmq/lib/riemann/tools/rabbitmq.rb

@@ -0,0 +1,269 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+module Riemann
+  module Tools
+    class Rabbitmq
+      include Riemann::Tools
+
+      require 'faraday'
+      require 'json'
+      require 'uri'
+
+      opt :read_timeout, 'Faraday read timeout', type: :int, default: 2
+      opt :open_timeout, 'Faraday open timeout', type: :int, default: 1
+
+      opt :monitor_user, 'RabbitMQ monitoring user', type: :string
+      opt :monitor_pass, 'RabbitMQ monitoring user password', type: :string
+      opt :monitor_port, 'RabbitMQ monitoring port', type: :int, default: 15_672
+      opt :monitor_host, 'RabbitMQ monitoring host', type: :string, default: 'localhost'
+      opt :monitor_use_tls, 'RabbitMQ use tls', type: :bool, default: false
+
+      opt :max_queue_size, 'max number of items in a queue that is acceptable', type: :int, default: 1_000_000
+      opt :ignore_max_size_queues, "A regular expression to match queues that shouldn't be size-checked", type: :string
+
+      opt :node, 'Specify a node to monitor', type: :strings
+
+      def base_url
+        protocol = 'http'
+        protocol = 'https' if options[:monitor_use_tls] && (options[:monitor_use_tls] == true)
+        "#{protocol}://#{options[:monitor_user]}:#{options[:monitor_pass]}@#{options[:monitor_host]}:#{options[:monitor_port]}/api"
+      end
+
+      def overview_url
+        "#{base_url}/overview"
+      end
+
+      def node_url(node)
+        "#{base_url}/nodes/#{node}"
+      end
+
+      def queues_url
+        "#{base_url}/queues"
+      end
+
+      def event_host
+        options[:event_host] || :monitor_host
+      end
+
+      def safe_get(uri, event_host)
+        # Handle connection timeouts
+        response = nil
+        begin
+          connection = Faraday.new(uri)
+          response = connection.get do |req|
+            req.options[:timeout] = options[:read_timeout]
+            req.options[:open_timeout] = options[:open_timeout]
+          end
+          report(
+            host: event_host,
+            service: 'rabbitmq monitoring',
+            state: 'ok',
+            description: 'Monitoring operational',
+          )
+        rescue StandardError => e
+          report(
+            host: event_host,
+            service: 'rabbitmq monitoring',
+            state: 'critical',
+            description: "HTTP connection error: #{e.class} - #{e.message}",
+          )
+        end
+        response
+      end
+
+      def check_queues
+        response = safe_get(queues_url, event_host)
+        max_size_check_filter = (Regexp.new(options[:ignore_max_size_queues]) if options[:ignore_max_size_queues])
+
+        return if response.nil?
+
+        if response.status != 200
+          report(
+            host: event_host,
+            service: 'rabbitmq.queue',
+            state: 'critical',
+            description: "HTTP connection error to /api/queues: #{response.status} - #{response.body}",
+          )
+        else
+          report(
+            host: event_host,
+            service: 'rabbitmq.queue',
+            state: 'ok',
+            description: 'HTTP connection ok',
+          )
+
+          json = JSON.parse(response.body)
+
+          json.each do |queue|
+            svc = "rabbitmq.queue.#{queue['vhost']}.#{queue['name']}"
+            errs = []
+
+            errs << 'Queue has jobs but no consumers' if !queue['messages_ready'].nil? && (queue['messages_ready']).positive? && (queue['consumers']).zero?
+
+            errs << "Queue has #{queue['messages_ready']} jobs" if (max_size_check_filter.nil? || queue['name'] !~ (max_size_check_filter)) && !queue['messages_ready'].nil? && (queue['messages_ready'] > options[:max_queue_size])
+
+            if errs.empty?
+              report(
+                host: event_host,
+                service: svc,
+                state: 'ok',
+                description: 'Queue is looking good',
+              )
+            else
+              report(
+                host: event_host,
+                service: svc,
+                state: 'critical',
+                description: errs.join('; '),
+              )
+            end
+
+            stats = (queue['message_stats'] || {}).merge(
+              'messages'                        => queue['messages'],
+              'messages_details'                => queue['messages_details'],
+              'messages_ready'                  => queue['messages_ready'],
+              'messages_ready_details'          => queue['messages_ready_details'],
+              'messages_unacknowledged'         => queue['messages_unacknowledged'],
+              'messages_unacknowledged_details' => queue['messages_unacknowledged_details'],
+              'consumers'                       => queue['consumers'],
+              'memory'                          => queue['memory'],
+            )
+
+            stats.each_pair do |k, v|
+              service = "#{svc}.#{k}"
+              metric = if k =~ (/details$/) && !v.nil?
+                         v['rate']
+                       else
+                         v
+                       end
+
+              # TODO: Set state via thresholds which can be configured
+
+              report(
+                host: event_host,
+                service: service,
+                metric: metric,
+                description: 'RabbitMQ monitor',
+              )
+            end
+          end
+        end
+      end
+
+      def check_overview
+        uri = URI(overview_url)
+        response = safe_get(uri, event_host)
+
+        return if response.nil?
+
+        json = JSON.parse(response.body)
+
+        if response.status != 200
+          report(
+            host: event_host,
+            service: 'rabbitmq',
+            state: 'critical',
+            description: "HTTP connection error: #{response.status} - #{response.body}",
+          )
+        else
+          report(
+            host: event_host,
+            service: 'rabbitmq monitoring',
+            state: 'ok',
+            description: 'HTTP connection ok',
+          )
+
+          %w[message_stats queue_totals object_totals].each do |stat|
+            # NOTE: / BUG ?
+            # Brand new servers can have blank message stats. Is this ok?
+            # I can't decide.
+            next if json[stat].empty?
+
+            json[stat].each_pair do |k, v|
+              service = "rabbitmq.#{stat}.#{k}"
+              metric = if k =~ /details$/
+                         v['rate']
+                       else
+                         v
+                       end
+
+              # TODO: Set state via thresholds which can be configured
+
+              report(
+                host: event_host,
+                service: service,
+                metric: metric,
+                description: 'RabbitMQ monitor',
+              )
+            end
+          end
+        end
+      end
+
+      def check_node
+        opts[:node].each do |n|
+          uri = URI(node_url(n))
+          response = safe_get(uri, event_host)
+
+          break if response.nil?
+
+          if response.status != 200
+            if response.status == 404
+              report(
+                host: event_host,
+                service: "rabbitmq.node.#{n}",
+                state: 'critical',
+                description: 'Node was not found in the cluster',
+              )
+            else
+              report(
+                host: event_host,
+                service: "rabbitmq.node.#{n}",
+                state: 'critical',
+                description: "HTTP error: #{response.status} - #{response.body}",
+              )
+            end
+            break
+          end
+
+          json = JSON.parse(response.body)
+
+          if json['mem_alarm']
+            report(
+              host: event_host,
+              service: "rabbitmq.node.#{n}",
+              state: 'critical',
+              description: 'Memory alarm has triggered; job submission throttled',
+            )
+            break
+          end
+
+          if json['disk_free_alarm']
+            report(
+              host: event_host,
+              service: "rabbitmq.node.#{n}",
+              state: 'critical',
+              description: 'Disk free alarm has triggered; job submission throttled',
+            )
+            break
+          end
+
+          report(
+            host: event_host,
+            service: "rabbitmq.node.#{n}",
+            state: 'ok',
+            description: 'Node looks OK to me',
+          )
+        end
+      end
+
+      def tick
+        check_overview
+        check_node if opts[:node]
+        check_queues
+      end
+    end
+  end
+end

data/tools/riemann-riak/Rakefile

@@ -11,10 +11,8 @@ ENV['COPYFILE_DISABLE'] = 'true'
 
 # Gemspec
 gemspec = Gem::Specification.new do |s|
-  s.rubyforge_project = 'riemann-riak'
-
   s.name = 'riemann-riak'
-  s.version = '0.1.2'
+  s.version = '0.1.4'
   s.author = 'Kyle Kingsbury'
   s.email = 'aphyr@aphyr.com'
   s.homepage = 'https://github.com/riemann/riemann-tools'
@@ -22,14 +20,13 @@ gemspec = Gem::Specification.new do |s|
   s.summary = 'Submits riak stats to riemann.'
   s.license = 'MIT'
 
-  s.add_dependency 'riemann-tools', '>= 0.2.13'
-  s.add_dependency 'yajl-ruby', '>= 1.1.0'
+  s.add_runtime_dependency 'riemann-tools', '~> 1.0', '>= 1.1.1'
+  s.add_runtime_dependency 'yajl-ruby', '~> 1.4', '>= 1.4.3'
 
-  s.files = FileList['bin/*', 'LICENSE', 'README.md'].to_a
+  s.files = FileList['bin/*', 'lib/**/*.rb', 'LICENSE', 'README.md'].to_a
   s.executables |= Dir.entries('bin/')
-  s.has_rdoc = false
 
-  s.required_ruby_version = '>= 1.8.7'
+  s.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
 end
 
 Gem::PackageTask.new gemspec do |p|

data/tools/riemann-riak/bin/riemann-riak

@@ -1,323 +1,8 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require 'English'
 Process.setproctitle($PROGRAM_NAME)
 
-# Forwards information on a Riak node to Riemann.
-
-require 'riemann/tools'
-
-module Riemann
-  module Tools
-    class Riak
-      include Riemann::Tools
-      require 'net/http'
-      require 'net/https'
-      require 'yajl/json_gem'
-
-      opt :riak_host, 'Riak host for stats <IP> or SSL http(s)://<IP>', default: Socket.gethostname
-      opt :data_dir, 'Riak data directory', default: '/var/lib/riak'
-      opt :stats_port, 'Riak HTTP port for stats', default: 8098
-      opt :stats_path, 'Riak HTTP stats path', default: '/stats'
-      opt :node_name, 'Riak erlang node name', default: "riak@#{Socket.gethostname}"
-      opt :cookie, 'Riak cookie to use', default: 'riak'
-
-      opt :get_50_warning, 'FSM 50% get time warning threshold (ms)', default: 1000
-      opt :put_50_warning, 'FSM 50% put time warning threshold (ms)', default: 1000
-      opt :get_95_warning, 'FSM 95% get time warning threshold (ms)', default: 2000
-      opt :put_95_warning, 'FSM 95% put time warning threshold (ms)', default: 2000
-      opt :get_99_warning, 'FSM 99% get time warning threshold (ms)', default: 10_000
-      opt :put_99_warning, 'FSM 99% put time warning threshold (ms)', default: 10_000
-
-      def initialize
-        detect_features
-
-        @httpstatus = true
-
-        begin
-          uri = URI.parse(opts[:riak_host])
-          uri.host = opts[:riak_host] if uri.host.nil?
-          http = Net::HTTP.new(uri.host, opts[:stats_port])
-          http.use_ssl = uri.scheme == 'https'
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl?
-          http.start do |h|
-            h.get opts[:stats_path]
-          end
-        rescue StandardError => _e
-          @httpstatus = false
-        end
-
-        # we're going to override the emulator setting to allow users to
-        # dynamically input the cookie
-        # this is done only once - hopefully it doesn't get overridden.
-        ENV['ERL_AFLAGS'] = "-setcookie #{opts[:cookie]}"
-      end
-
-      # Identifies whether escript and riak-admin are installed
-      def detect_features
-        @escript = true # Whether escript is present on this machine
-        @riakadmin = true # Whether riak-admin is present
-
-        @escript = false if `which escript` =~ /^\s*$/
-
-        @riakadmin = false if `which riak-admin` =~ /^\s*$/
-      end
-
-      def check_ring
-        str = if @escript
-                `#{__dir__}/riemann-riak-ring #{opts[:node_name]}`.chomp
-              elsif @riakadmin
-                `riak-admin ringready`
-              end
-
-        return if str.nil?
-
-        if str =~ /^TRUE/
-          report(
-            host: opts[:riak_host],
-            service: 'riak ring',
-            state: 'ok',
-            description: str,
-          )
-        else
-          report(
-            host: opts[:riak_host],
-            service: 'riak ring',
-            state: 'warning',
-            description: str,
-          )
-        end
-      end
-
-      def check_keys
-        keys = `#{__dir__}/riemann-riak-keys #{opts[:node_name]}`.chomp
-        if keys =~ /^\d+$/
-          report(
-            host: opts[:riak_host],
-            service: 'riak keys',
-            state: 'ok',
-            metric: keys.to_i,
-            description: keys,
-          )
-        else
-          report(
-            host: opts[:riak_host],
-            service: 'riak keys',
-            state: 'unknown',
-            description: keys,
-          )
-        end
-      end
-
-      def check_transfers
-        str = (`riak-admin transfers` if @riakadmin)
-
-        return if str.nil?
-
-        if str =~ /'#{opts[:node_name]}' waiting to handoff (\d+) partitions/
-          report(
-            host: opts[:riak_host],
-            service: 'riak transfers',
-            state: 'critical',
-            metric: Regexp.last_match(1).to_i,
-            description: "waiting to handoff #{Regexp.last_match(1)} partitions",
-          )
-        else
-          report(
-            host: opts[:riak_host],
-            service: 'riak transfers',
-            state: 'ok',
-            metric: 0,
-            description: 'No pending transfers',
-          )
-        end
-      end
-
-      def check_disk
-        gb = `du -Ls #{opts[:data_dir]}`.split(/\s+/).first.to_i / (1024.0**2)
-        report(
-          host: opts[:riak_host],
-          service: 'riak disk',
-          state: 'ok',
-          metric: gb,
-          description: "#{gb} GB in #{opts[:data_dir]}",
-        )
-      end
-
-      # Returns the riak stat for the given fsm type and percentile.
-      def fsm_stat(type, property, percentile)
-        "node_#{type}_fsm_#{property}_#{percentile == 50 ? 'median' : percentile}"
-      end
-
-      # Returns the alerts state for the given fsm.
-      def fsm_state(type, percentile, val)
-        limit = opts["#{type}_#{percentile}_warning".to_sym]
-        case val
-        when 0..limit
-          'ok'
-        when limit..limit * 2
-          'warning'
-        else
-          'critical'
-        end
-      end
-
-      # Get current stats via HTTP
-      def stats_http
-        begin
-          uri = URI.parse(opts[:riak_host])
-          uri.host = opts[:riak_host] if uri.host.nil?
-          http = Net::HTTP.new(uri.host, opts[:stats_port])
-          http.use_ssl = uri.scheme == 'https'
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl?
-          res = http.start do |h|
-            h.get opts[:stats_path]
-          end
-        rescue StandardError => e
-          report(
-            host: opts[:riak_host],
-            service: 'riak',
-            state: 'critical',
-            description: "error fetching #{opts[:riak_host]}:#{opts[:stats_port]} #{e.class}, #{e.message}",
-          )
-          raise
-        end
-
-        if res.code.to_i == 200
-          JSON.parse(res.body)
-        else
-          report(
-            host: opts[:riak_host],
-            service: 'riak',
-            state: 'critical',
-            description: "stats returned HTTP #{res.code}:\n\n#{res.body}",
-          )
-          raise "Can't fetch stats via HTTP: #{res.core}:\n\n#{res.body}"
-        end
-      end
-
-      # Get current stats via riak-admin
-      def stats_riak_admin
-        str = `riak-admin status`
-        raise 'riak-admin failed' unless $CHILD_STATUS == 0
-
-        Hash[str.split(/\n/).map { |i| i.split(/ : /) }]
-      end
-
-      # Get current stats as a hash
-      def stats
-        if @httpstatus
-          stats_http
-        elsif @riakadmin
-          stats_riak_admin
-        else
-          report(
-            host: opts[:riak_host],
-            service: 'riak',
-            state: 'critical',
-            description: 'No mechanism for fetching Riak stats: neither HTTP nor riak-admin available.',
-          )
-          raise 'No mechanism for fetching Riak stats: neither HTTP nor riak-admin available.'
-        end
-      end
-
-      def core_services
-        %w[vnode_gets
-           vnode_puts
-           node_gets
-           node_puts
-           node_gets_set
-           node_puts_set
-           read_repairs]
-      end
-
-      def fsm_types
-        [{ 'get' => 'time' }, { 'put' => 'time' },
-         { 'get' => 'set_objsize' },]
-      end
-
-      def fsm_percentiles
-        [50, 95, 99]
-      end
-
-      # Reports current stats to Riemann
-      def check_stats
-        begin
-          stats = self.stats
-        rescue StandardError => e
-          event = {
-            state: 'critical',
-            description: e.message,
-            host: opts[:riak_host],
-          }
-          # Report errors
-          report(event.merge(service: 'riak'))
-          core_services.each do |s|
-            report(event.merge(service: "riak #{s}"))
-          end
-          fsm_types.each do |typespec|
-            typespec.each do |type, prop|
-              fsm_percentiles.each do |percentile|
-                report(event.merge(service: "riak #{type} #{prop} #{percentile}"))
-              end
-            end
-          end
-          return
-        end
-
-        # Riak itself
-        report(
-          host: opts[:riak_host],
-          service: 'riak',
-          state: 'ok',
-        )
-
-        # Gets/puts/rr
-        core_services.each do |s|
-          report(
-            host: opts[:riak_host],
-            service: "riak #{s}",
-            state: 'ok',
-            metric: stats[s].to_i / 60.0,
-            description: "#{stats[s].to_i / 60.0}/sec",
-          )
-        end
-
-        # FSMs
-        fsm_types.each do |typespec|
-          typespec.each do |type, prop|
-            fsm_percentiles.each do |percentile|
-              val = stats[fsm_stat(type, prop, percentile)].to_i || 0
-              val = 0 if val == 'undefined'
-              val /= 1000.0 if prop == 'time' # Convert us to ms
-              state = if prop == 'time'
-                        fsm_state(type, percentile, val)
-                      else
-                        'ok'
-                      end
-              report(
-                host: opts[:riak_host],
-                service: "riak #{type} #{prop} #{percentile}",
-                state: state,
-                metric: val,
-                description: "#{val} ms",
-              )
-            end
-          end
-        end
-      end
-
-      def tick
-        # This can utterly destroy a cluster, so we disable
-        # check_keys
-        check_stats
-        check_ring
-        check_disk
-        check_transfers
-      end
-    end
-  end
-end
+require 'riemann/tools/riak'
 
 Riemann::Tools::Riak.run

data/tools/riemann-riak/bin/riemann-riak-keys

@@ -1,5 +1,4 @@
 #!/usr/bin/env escript
-Process.setproctitle($0)
 %%! -name riakstatuscheck@127.0.0.1 -hidden
 
 main([]) -> main(["riak@127.0.0.1"]);

data/tools/riemann-riak/bin/riemann-riak-ring

@@ -1,5 +1,4 @@
 #!/usr/bin/env escript
-Process.setproctitle($0)
 %%! -name riakstatuscheck@127.0.0.1 -hidden
 
 main([]) -> main(["riak@127.0.0.1"]);