riemann-tools 0.2.14 → 1.0.0

data/tools/riemann-rabbitmq/bin/riemann-rabbitmq

@@ -0,0 +1,269 @@
+#!/usr/bin/env ruby
+Process.setproctitle($0)
+
+require 'riemann/tools'
+
+class Riemann::Tools::Rabbitmq
+  include Riemann::Tools
+
+  require 'faraday'
+  require 'json'
+  require 'uri'
+
+
+  opt :read_timeout, 'Faraday read timeout', type: :int, default: 2
+  opt :open_timeout, 'Faraday open timeout', type: :int, default: 1
+
+  opt :monitor_user, 'RabbitMQ monitoring user', type: :string
+  opt :monitor_pass, 'RabbitMQ monitoring user password', type: :string
+  opt :monitor_port, 'RabbitMQ monitoring port', type: :int, default: 15672
+  opt :monitor_host, 'RabbitMQ monitoring host', type: :string, default: "localhost"
+  opt :monitor_use_tls, 'RabbitMQ use tls', type: :bool, default: false
+
+  opt :max_queue_size, "max number of items in a queue that is acceptable", type: :int, default: 1_000_000
+  opt :ignore_max_size_queues, "A regular expression to match queues that shouldn't be size-checked", type: :string
+
+  opt :node, "Specify a node to monitor", type: :strings
+
+  def base_url
+    protocol = "http"
+    if (options[:monitor_use_tls]) && (options[:monitor_use_tls]==true)
+      protocol = "https"
+    end
+    "#{protocol}://#{options[:monitor_user]}:#{options[:monitor_pass]}@#{options[:monitor_host]}:#{options[:monitor_port]}/api"
+  end
+
+  def overview_url
+    "#{base_url}/overview"
+  end
+
+  def node_url(n)
+    "#{base_url}/nodes/#{n}"
+  end
+
+  def queues_url
+    "#{base_url}/queues"
+  end
+
+  def event_host
+    if options[:event_host]
+      return options[:event_host]
+    else
+      return options[:monitor_host]
+    end
+  end
+
+  def safe_get(uri, event_host)
+      # Handle connection timeouts
+      response = nil
+      begin
+        connection = Faraday.new(uri)
+        response = connection.get do |req|
+          req.options[:timeout] = options[:read_timeout]
+          req.options[:open_timeout] = options[:open_timeout]
+        end
+        report(:host => event_host,
+          :service => "rabbitmq monitoring",
+          :state   => 'ok',
+          :description => "Monitoring operational"
+        )
+      rescue => e
+        report(:host => event_host,
+          :service => "rabbitmq monitoring",
+          :state => "critical",
+          :description => "HTTP connection error: #{e.class} - #{e.message}"
+        )
+      end
+      response
+  end
+
+  def check_queues
+    response = safe_get(queues_url, event_host)
+    max_size_check_filter = if options[:ignore_max_size_queues]
+      Regexp.new(options[:ignore_max_size_queues])
+    else
+      nil
+    end
+
+    return if response.nil?
+
+    json = JSON.parse(response.body)
+
+    if response.status != 200
+        report(:host => event_host,
+          :service => "rabbitmq.queue",
+          :state => "critical",
+          :description => "HTTP connection error to /api/queues: #{response.status} - #{response.body}"
+        )
+    else
+      report(:host => event_host,
+        :service => "rabbitmq.queue",
+        :state => "ok",
+        :description => "HTTP connection ok"
+      )
+
+      json = JSON.parse(response.body)
+
+      json.each do |queue|
+        svc = "rabbitmq.queue.#{queue['vhost']}.#{queue['name']}"
+        errs = []
+
+        if queue['messages_ready']!=nil and queue['messages_ready'] > 0 and queue['consumers'] == 0
+          errs << "Queue has jobs but no consumers"
+        end
+
+        if (max_size_check_filter.nil? or queue['name'] !~ max_size_check_filter) and queue['messages_ready']!=nil and queue['messages_ready'] > options[:max_queue_size]
+          errs << "Queue has #{queue['messages_ready']} jobs"
+        end
+
+        if errs.empty?
+          report(:host    => event_host,
+                 :service => svc,
+                 :state   => "ok",
+                 :description => "Queue is looking good"
+          )
+        else
+          report(:host    => event_host,
+                 :service => svc,
+                 :state   => "critical",
+                 :description => errs.join("; ")
+          )
+        end
+
+        stats = (queue['message_stats'] || {}).merge(
+                  'messages'                        => queue['messages'],
+                  'messages_details'                => queue['messages_details'],
+                  'messages_ready'                  => queue['messages_ready'],
+                  'messages_ready_details'          => queue['messages_ready_details'],
+                  'messages_unacknowledged'         => queue['messages_unacknowledged'],
+                  'messages_unacknowledged_details' => queue['messages_unacknowledged_details'],
+                  'consumers'                       => queue['consumers'],
+                  'memory'                          => queue['memory'],
+                )
+
+        stats.each_pair do |k,v|
+          service = "#{svc}.#{k}"
+          if k =~ /details$/ and v!=nil
+            metric = v['rate']
+          else
+            metric = v
+          end
+
+          # TODO: Set state via thresholds which can be configured
+
+          report(:host => event_host,
+                 :service => service,
+                 :metric => metric,
+                 :description => "RabbitMQ monitor"
+          )
+        end
+      end
+    end
+  end
+
+  def check_overview
+    uri = URI(overview_url)
+    response = safe_get(uri, event_host)
+
+    return if response.nil?
+
+    json = JSON.parse(response.body)
+
+    if response.status != 200
+        report(:host => event_host,
+          :service => "rabbitmq",
+          :state => "critical",
+          :description => "HTTP connection error: #{response.status} - #{response.body}"
+        )
+    else
+      report(:host => event_host,
+        :service => "rabbitmq monitoring",
+        :state => "ok",
+        :description => "HTTP connection ok"
+      )
+
+      %w( message_stats queue_totals object_totals ).each do |stat|
+        # NOTE / BUG ?
+        # Brand new servers can have blank message stats. Is this ok?
+        # I can't decide.
+        next if json[stat].empty?
+        json[stat].each_pair do |k,v|
+          service = "rabbitmq.#{stat}.#{k}"
+          if k =~ /details$/
+            metric = v['rate']
+          else
+            metric = v
+          end
+
+          # TODO: Set state via thresholds which can be configured
+
+          report(:host => event_host,
+                 :service => service,
+                 :metric => metric,
+                 :description => "RabbitMQ monitor"
+                )
+        end
+      end
+    end
+  end
+
+  def check_node
+    opts[:node].each do |n|
+      uri = URI(node_url(n))
+      response = safe_get(uri, event_host)
+
+      return if response.nil?
+
+      if response.status != 200
+        if response.status == 404
+          report(:host => event_host,
+            :service => "rabbitmq.node.#{n}",
+            :state => "critical",
+            :description => "Node was not found in the cluster"
+          )
+        else
+          report(:host => event_host,
+            :service => "rabbitmq.node.#{n}",
+            :state => "critical",
+            :description => "HTTP error: #{response.status} - #{response.body}"
+          )
+        end
+        return
+      end
+
+      json = JSON.parse(response.body)
+
+      if json['mem_alarm']
+        report(:host => event_host,
+          :service => "rabbitmq.node.#{n}",
+          :state     => "critical",
+          :description => "Memory alarm has triggered; job submission throttled"
+        )
+        return
+      end
+
+      if json['disk_free_alarm']
+        report(:host => event_host,
+          :service => "rabbitmq.node.#{n}",
+          :state     => "critical",
+          :description => "Disk free alarm has triggered; job submission throttled"
+        )
+        return
+      end
+
+      report(:host => event_host,
+        :service => "rabbitmq.node.#{n}",
+        :state   => "ok",
+        :description => "Node looks OK to me"
+      )
+    end
+  end
+
+  def tick
+    check_overview
+    check_node if opts[:node]
+    check_queues
+  end
+end
+Riemann::Tools::Rabbitmq.run
+

data/tools/riemann-riak/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2011 Kyle Kingsbury
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/tools/riemann-riak/README.md

@@ -0,0 +1,10 @@
+# Riemann Riak
+
+Gathers Riak statistics and submits them to Riemann.
+
+## Getting started
+
+```
+gem install riemann-riak
+riemann-riak --help
+```

data/tools/riemann-riak/Rakefile.rb

@@ -0,0 +1,34 @@
+require 'rubygems'
+require 'rubygems/package_task'
+require 'rdoc/task'
+require 'find'
+
+# Don't include resource forks in tarballs on Mac OS X.
+ENV['COPY_EXTENDED_ATTRIBUTES_DISABLE'] = 'true'
+ENV['COPYFILE_DISABLE'] = 'true'
+
+# Gemspec
+gemspec = Gem::Specification.new do |s|
+  s.rubyforge_project = 'riemann-riak'
+
+  s.name = 'riemann-riak'
+  s.version = '0.1.2'
+  s.author = 'Kyle Kingsbury'
+  s.email = 'aphyr@aphyr.com'
+  s.homepage = 'https://github.com/riemann/riemann-tools'
+  s.platform = Gem::Platform::RUBY
+  s.summary = 'Submits riak stats to riemann.'
+  s.license = 'MIT'
+
+  s.add_dependency 'riemann-tools', '>= 0.2.13'
+  s.add_dependency 'yajl-ruby', '>= 1.1.0'
+
+  s.files = FileList['bin/*', 'LICENSE', 'README.md'].to_a
+  s.executables |= Dir.entries('bin/')
+  s.has_rdoc = false
+
+  s.required_ruby_version = '>= 1.8.7'
+end
+
+Gem::PackageTask.new gemspec do |p|
+end

data/tools/riemann-riak/bin/riemann-riak

@@ -0,0 +1,331 @@
+#!/usr/bin/env ruby
+Process.setproctitle($0)
+
+# Forwards information on a Riak node to Riemann.
+
+require 'riemann/tools'
+
+class Riemann::Tools::Riak
+  include Riemann::Tools
+  require 'net/http'
+  require 'net/https'
+  require 'yajl/json_gem'
+
+  opt :riak_host, "Riak host for stats <IP> or SSL http(s)://<IP>", :default => Socket.gethostname
+  opt :data_dir, "Riak data directory", :default => '/var/lib/riak'
+  opt :stats_port, "Riak HTTP port for stats", :default => 8098
+  opt :stats_path, "Riak HTTP stats path", :default => '/stats'
+  opt :node_name, "Riak erlang node name", :default => "riak@#{Socket.gethostname}"
+  opt :cookie, "Riak cookie to use", :default => "riak"
+
+  opt :get_50_warning, "FSM 50% get time warning threshold (ms)", :default => 1000
+  opt :put_50_warning, "FSM 50% put time warning threshold (ms)", :default => 1000
+  opt :get_95_warning, "FSM 95% get time warning threshold (ms)", :default => 2000
+  opt :put_95_warning, "FSM 95% put time warning threshold (ms)", :default => 2000
+  opt :get_99_warning, "FSM 99% get time warning threshold (ms)", :default => 10000
+  opt :put_99_warning, "FSM 99% put time warning threshold (ms)", :default => 10000
+
+  def initialize
+    detect_features
+
+    @httpstatus = true
+
+    begin
+      uri = URI.parse(opts[:riak_host])
+      if uri.host == nil
+        uri.host = opts[:riak_host]
+      end
+      http = Net::HTTP.new(uri.host, opts[:stats_port])
+      http.use_ssl = uri.scheme == 'https'
+      if http.use_ssl?
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+      http.start do |h|
+        h.get opts[:stats_path]
+      end
+    rescue => _e
+      @httpstatus = false
+    end
+
+    # we're going to override the emulator setting to allow users to
+    # dynamically input the cookie
+    # this is done only once - hopefully it doesn't get overridden.
+    ENV['ERL_AFLAGS'] = "-setcookie #{opts[:cookie]}"
+  end
+
+  # Identifies whether escript and riak-admin are installed
+  def detect_features
+    @escript = true # Whether escript is present on this machine
+    @riakadmin = true # Whether riak-admin is present
+
+    if `which escript` =~ /^\s*$/
+      @escript = false
+    end
+
+    if `which riak-admin` =~ /^\s*$/
+      @riakadmin = false
+    end
+  end
+
+  def check_ring
+    str = if @escript
+      str = `#{File.expand_path(File.dirname(__FILE__))}/riemann-riak-ring #{opts[:node_name]}`.chomp
+    elsif @riakadmin
+      str = `riak-admin ringready`
+    else
+      nil
+    end
+
+    return if str.nil?
+
+    if str =~ /^TRUE/
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak ring',
+        :state => 'ok',
+        :description => str
+      )
+    else
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak ring',
+        :state => 'warning',
+        :description => str
+      )
+    end
+  end
+
+  def check_keys
+    keys = `#{File.expand_path(File.dirname(__FILE__))}/riemann-riak-keys #{opts[:node_name]}`.chomp
+    if keys =~ /^\d+$/
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak keys',
+        :state => 'ok',
+        :metric => keys.to_i,
+        :description => keys
+      )
+    else
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak keys',
+        :state => 'unknown',
+        :description => keys
+      )
+    end
+  end
+
+  def check_transfers
+    str = if @riakadmin
+      `riak-admin transfers`
+    else
+      nil
+    end
+
+    return if str.nil?
+
+    if str =~ /'#{opts[:node_name]}' waiting to handoff (\d+) partitions/
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak transfers',
+        :state   => 'critical',
+        :metric  => $1.to_i,
+        :description => "waiting to handoff #{$1} partitions"
+      )
+    else
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak transfers',
+        :state   => 'ok',
+        :metric  => 0,
+        :description => "No pending transfers"
+      )
+    end
+  end
+
+  def check_disk
+    gb = `du -Ls #{opts[:data_dir]}`.split(/\s+/).first.to_i / (1024.0**2)
+    report(
+      :host => opts[:riak_host],
+      :service => 'riak disk',
+      :state => 'ok',
+      :metric => gb,
+      :description => "#{gb} GB in #{opts[:data_dir]}"
+    )
+  end
+
+  # Returns the riak stat for the given fsm type and percentile.
+  def fsm_stat(type, property, percentile)
+    "node_#{type}_fsm_#{property}_#{percentile == 50 ? 'median' : percentile}"
+  end
+
+  # Returns the alerts state for the given fsm.
+  def fsm_state(type, percentile, val)
+    limit = opts["#{type}_#{percentile}_warning".to_sym]
+    case val
+    when 0 .. limit
+      'ok'
+    when limit .. limit * 2
+      'warning'
+    else
+      'critical'
+    end
+  end
+
+  # Get current stats via HTTP
+  def stats_http
+    begin
+      uri = URI.parse(opts[:riak_host])
+      if uri.host == nil
+        uri.host = opts[:riak_host]
+      end
+      http = Net::HTTP.new(uri.host, opts[:stats_port])
+      http.use_ssl = uri.scheme == 'https'
+      if http.use_ssl?
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+      res = http.start do |h|
+        h.get opts[:stats_path]
+      end
+    rescue => e
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak',
+        :state => 'critical',
+        :description => "error fetching #{opts[:riak_host]}:#{opts[:stats_port]} #{e.class}, #{e.message}"
+      )
+      raise
+    end
+
+    if res.code.to_i == 200
+      return JSON.parse(res.body)
+    else
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak',
+        :state => 'critical',
+        :description => "stats returned HTTP #{res.code}:\n\n#{res.body}"
+      )
+      raise "Can't fetch stats via HTTP: #{res.core}:\n\n#{res.body}"
+    end
+  end
+
+  # Get current stats via riak-admin
+  def stats_riak_admin
+    str = `riak-admin status`
+    raise "riak-admin failed" unless $? == 0
+    Hash[str.split(/\n/).map{|i| i.split(/ : /)}]
+  end
+
+  # Get current stats as a hash
+  def stats
+    if @httpstatus
+      stats_http
+    elsif @riakadmin
+      stats_riak_admin
+    else
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak',
+        :state => 'critical',
+        :description => "No mechanism for fetching Riak stats: neither HTTP nor riak-admin available."
+      )
+      raise "No mechanism for fetching Riak stats: neither HTTP nor riak-admin available."
+    end
+  end
+
+  def core_services
+    ['vnode_gets',
+     'vnode_puts',
+     'node_gets',
+     'node_puts',
+     'node_gets_set',
+     'node_puts_set',
+     'read_repairs']
+  end
+
+  def fsm_types
+    [{'get' => 'time'}, {'put' => 'time'},
+     {'get' => 'set_objsize'}]
+  end
+
+  def fsm_percentiles
+    [50, 95, 99]
+  end
+
+  # Reports current stats to Riemann
+  def check_stats
+    begin
+      stats = self.stats
+    rescue => e
+      event = {:state => 'critical',
+               :description => e.message,
+               :host => opts[:riak_host]}
+      # Report errors
+      report(event.merge(:service => 'riak'))
+      core_services.each do |s|
+        report(event.merge(:service => "riak #{s}"))
+      end
+      fsm_types.each do |typespec|
+        typespec.each do |type, prop|
+          fsm_percentiles.each do |percentile|
+            report(event.merge(:service => "riak #{type} #{prop} #{percentile}"))
+          end
+        end
+      end
+      return
+    end
+
+    # Riak itself
+    report(
+      :host => opts[:riak_host],
+      :service => 'riak',
+      :state => 'ok'
+    )
+
+    # Gets/puts/rr
+    core_services.each do |s|
+      report(
+        :host => opts[:riak_host],
+        :service => "riak #{s}",
+        :state => 'ok',
+        :metric => stats[s].to_i/60.0,
+        :description => "#{stats[s].to_i/60.0}/sec"
+      )
+    end
+
+    # FSMs
+    fsm_types.each do |typespec|
+      typespec.each do |type, prop|
+        fsm_percentiles.each do |percentile|
+          val = stats[fsm_stat(type, prop, percentile)].to_i || 0
+          val = 0 if val == 'undefined'
+          val /= 1000.0 if prop == 'time' # Convert us to ms
+          if prop == 'time'
+            state = fsm_state(type,  percentile, val)
+          else
+            state = "ok"
+          end
+          report(
+            :host => opts[:riak_host],
+            :service => "riak #{type} #{prop} #{percentile}",
+            :state => state,
+            :metric => val,
+            :description => "#{val} ms"
+          )
+        end
+      end
+    end
+  end
+
+  def tick
+    # This can utterly destroy a cluster, so we disable
+    # check_keys
+    check_stats
+    check_ring
+    check_disk
+    check_transfers
+  end
+end
+
+Riemann::Tools::Riak.run

data/tools/riemann-riak/bin/riemann-riak-keys

@@ -0,0 +1,13 @@
+#!/usr/bin/env escript
+Process.setproctitle($0)
+%%! -name riakstatuscheck@127.0.0.1 -hidden
+
+main([]) -> main(["riak@127.0.0.1"]);
+main([Node]) ->
+  io:format("~w\n", [
+    lists:foldl(
+      fun({_VNode, Count}, Sum) -> Sum + Count end,
+      0,
+      rpc:call(list_to_atom(Node), riak_kv_bitcask_backend, key_counts, [])
+    )
+  ]).

data/tools/riemann-riak/bin/riemann-riak-ring

@@ -0,0 +1,9 @@
+#!/usr/bin/env escript
+Process.setproctitle($0)
+%%! -name riakstatuscheck@127.0.0.1 -hidden
+
+main([]) -> main(["riak@127.0.0.1"]);
+main([Node]) ->
+  io:format("~p\n", [
+    rpc:call(list_to_atom(Node), riak_kv_console, ringready, [[]])
+  ]).

data/tools/riemann-riak/riak_status/key_count.erl

@@ -0,0 +1,13 @@
+#!/usr/bin/env escript
+Process.setproctitle($0)
+%%! -name riakstatuscheck -setcookie riak -hidden
+
+main([]) -> main(["riak@127.0.0.1"]);
+main([Node]) ->
+  io:format("~w\n", [
+    lists:foldl(
+      fun({_VNode, Count}, Sum) -> Sum + Count end,
+      0,
+      rpc:call(list_to_atom(Node), riak_kv_bitcask_backend, key_counts, [])
+    )
+  ]).