riemann-tools.haf 0.1.9

data/bin/riemann-resmon

@@ -0,0 +1,103 @@
+#!/usr/bin/env ruby
+
+require File.expand_path('../../lib/riemann/tools', __FILE__)
+
+class Riemann::Tools::Resmon
+  include Riemann::Tools
+  require 'nokogiri'
+  require 'faraday'
+
+  opt :resmon_hostfile, 'File with hostnames running Resmon (one URI per line)', type: :string
+  opt :read_timeout, 'Faraday read timeout', type: :int, default: 2
+  opt :open_timeout, 'Faraday open timeout', type: :int, default: 1
+  opt :fqdn, 'Use FQDN for event host'
+
+
+  def initialize
+    @hosts = File.read(options[:resmon_hostfile]).split("\n")
+    super
+  end
+
+
+  # Work out the hostname to submit with the event
+  def get_event_host(host)
+    unless options[:fqdn]
+      return host.split('.')[0]
+    end
+    return host
+  end
+
+  # Handles HTTP connections and GET requests safely
+  def safe_get(uri, event_host)
+      # Handle connection timeouts
+      response = nil
+      begin
+        connection = Faraday.new(uri)
+        response = connection.get do |req|
+          req.options[:timeout] = options[:read_timeout]
+          req.options[:open_timeout] = options[:open_timeout]
+        end
+      rescue => e
+        report(:host => event_host,
+          :service => "resmon",
+          :state => "critical",
+          :description => "HTTP connection error: #{e.class} - #{e.message}"
+        )
+      end
+      response
+  end
+
+  def tick
+    @hosts.each do |host|
+
+      uri = URI(host)
+      event_host = get_event_host(uri.host)
+
+      response = safe_get(uri, event_host)
+      next if response.nil?
+
+      # Handle non-200 responses
+      if response.status != 200
+        report(:host => event_host,
+          :service => "resmon",
+          :state => "critical",
+          :description => "HTTP connection error: #{response.status} - #{response.body}"
+        )
+        next
+      else
+        report(:host => event_host,
+          :service => "resmon",
+          :state => "ok",
+          :description => "Resmon connection ok"
+        )
+        doc = Nokogiri::XML(response.body)
+      end
+
+      doc.xpath('//ResmonResults/ResmonResult').each do |result|
+        timestamp = result.xpath('last_update').first.text
+        result.xpath('metric').each do |metric|
+          hash = {
+            host: event_host,
+            service: "#{result.attributes['module'].value}`#{result.attributes['service'].value}`#{metric.attributes['name'].value}",
+            time: timestamp.to_i
+          }
+
+          case metric.attributes['type'].value
+            when /[iIlL]/
+              hash[:metric] = metric.text.to_i
+            when 'n'
+              hash[:metric] = metric.text.to_f
+            when 's'
+              hash[:description] = metric.text
+            when '0'
+              raise 'dunno what 0 is yet'
+          end
+
+          report(hash)
+        end
+      end
+    end
+  end
+end
+
+Riemann::Tools::Resmon.run

data/bin/riemann-riak

@@ -0,0 +1,237 @@
+#!/usr/bin/env ruby
+
+# Forwards information on a Riak node to Riemann.
+
+require File.expand_path('../../lib/riemann/tools', __FILE__)
+
+require 'net/http'
+require 'net/https'
+require 'yajl/json_gem'
+
+class Riemann::Tools::Riak
+  include Riemann::Tools
+
+  opt :riak_host, "Riak host for stats <IP> or SSL http(s)://<IP>", :default => Socket.gethostname
+  opt :data_dir, "Riak data directory", :default => '/var/lib/riak'
+  opt :stats_port, "Riak HTTP port for stats", :default => 8098
+  opt :stats_path, "Riak HTTP stats path", :default => '/stats'
+  opt :node_name, "Riak erlang node name", :default => "riak@#{Socket.gethostname}"
+  opt :cookie, "Riak cookie to use", :default => "riak"
+
+  opt :get_50_warning, "FSM 50% get time warning threshold (ms)", :default => 1000
+  opt :put_50_warning, "FSM 50% put time warning threshold (ms)", :default => 1000
+  opt :get_95_warning, "FSM 95% get time warning threshold (ms)", :default => 2000
+  opt :put_95_warning, "FSM 95% put time warning threshold (ms)", :default => 2000
+  opt :get_99_warning, "FSM 99% get time warning threshold (ms)", :default => 10000
+  opt :put_99_warning, "FSM 99% put time warning threshold (ms)", :default => 10000
+
+  def initialize
+    @escript = true
+    @riakadmin = true
+    @httpstatus = true
+
+    if `which escript` =~ /^\s*$/
+      @escript = false
+    end
+
+    if `which riak-admin` =~ /^\s*$/
+      @riakadmin = false
+    end
+
+    if
+      begin
+        uri = URI.parse(opts[:riak_host])
+      if uri.host == nil
+      uri.host = opts[:riak_host]
+      end
+        http = Net::HTTP.new(uri.host, opts[:stats_port])
+        http.use_ssl = uri.scheme == 'https'
+          if http.use_ssl?
+              http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          end
+        http.start do |http|
+        http.get opts[:stats_path]
+      end
+      rescue => e
+        @httpstatus = false
+      end
+    end
+    # we're going to override the emulator setting to allow users to
+    # dynamically input the cookie
+    # this is done only once - hopefully it doesn't get overridden.
+    ENV['ERL_AFLAGS'] = "-setcookie #{opts[:cookie]}"
+
+  end
+
+  def check_ring
+    if @escript
+      str = `#{File.expand_path(File.dirname(__FILE__))}/riemann-riak-ring #{opts[:node_name]}`.chomp
+    elsif @riakadmin
+      str = `riak-admin ringready`
+    end
+
+    if str =~ /^TRUE/
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak ring',
+        :state => 'ok',
+        :description => str
+      )
+    else
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak ring',
+        :state => 'warning',
+        :description => str
+      )
+    end
+  end
+
+  def check_keys
+    keys = `#{File.expand_path(File.dirname(__FILE__))}/riemann-riak-keys #{opts[:node_name]}`.chomp
+    if keys =~ /^\d+$/
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak keys',
+        :state => 'ok',
+        :metric => keys.to_i,
+        :description => keys
+      )
+    else
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak keys',
+        :state => 'unknown',
+        :description => keys
+      )
+    end
+  end
+
+  def check_disk
+    gb = `du -s #{opts[:data_dir]}`.split(/\s+/).first.to_i / (1024.0**2)
+    report(
+      :host => opts[:riak_host],
+      :service => 'riak disk',
+      :state => 'ok',
+      :metric => gb,
+      :description => "#{gb} GB in #{opts[:data_dir]}"
+    )
+  end
+
+  # Returns the riak stat for the given fsm type and percentile.
+  def fsm_stat(type, percentile)
+    "node_#{type}_fsm_time_#{percentile == 50 ? 'median' : percentile}"
+  end
+
+  # Returns the alerts state for the given fsm.
+  def fsm_state(type, percentile, val)
+    limit = opts["#{type}_#{percentile}_warning".to_sym]
+    case val
+    when 0 .. limit
+      'ok'
+    when limit .. limit * 2
+      'warning'
+    else
+      'critical'
+    end
+  end
+
+  def check_stats
+    if @httpstatus
+      begin
+        uri = URI.parse(opts[:riak_host])
+        if uri.host == nil
+          uri.host = opts[:riak_host]
+        end
+        http = Net::HTTP.new(uri.host, opts[:stats_port])
+        http.use_ssl = uri.scheme == 'https'
+        if http.use_ssl?
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+        res = http.start do |http|
+          http.get opts[:stats_path]
+      end
+      rescue => e
+        report(
+          :host => opts[:riak_host],
+          :service => 'riak',
+          :state => 'critical',
+          :description => "error fetching #{opts[:riak_host]}:#{opts[:stats_port]} #{e.class}, #{e.message}"
+        )
+        return
+      end
+
+      if res.code.to_i == 200
+        stats = JSON.parse(res.body)
+      else
+        report(
+          :host => opts[:riak_host],
+          :service => 'riak',
+          :state => 'critical',
+          :description => "stats returned HTTP #{res.code}:\n\n#{res.body}"
+        )
+        return
+      end
+    elsif @riakadmin
+      stats = Hash[`riak-admin status`.split(/\n/).map{|i| i.split(/ : /)}]
+    else
+        report(
+          :host => opts[:riak_host],
+          :service => 'riak',
+          :state => 'critical',
+          :description => "error fetching Riak stats"
+        )
+        return
+    end
+
+    report(
+      :host => opts[:riak_host],
+      :service => 'riak',
+      :state => 'ok'
+    )
+
+    # Gets/puts/rr
+    [
+      'vnode_gets',
+      'vnode_puts',
+      'node_gets',
+      'node_puts',
+      'read_repairs'
+    ].each do |s|
+      report(
+        :host => opts[:riak_host],
+        :service => "riak #{s}",
+        :state => 'ok',
+        :metric => stats[s].to_i/60.0,
+        :description => "#{stats[s].to_i/60.0}/sec"
+      )
+    end
+
+    # FSMs
+    ['get', 'put'].each do |type|
+      [50, 95, 99].each do |percentile|
+        val = stats[fsm_stat(type, percentile)].to_i || 0
+        val = 0 if val == 'undefined'
+        val /= 1000.0 # Convert us to ms
+        state = fsm_state(type, percentile, val)
+        report(
+          :host => opts[:riak_host],
+          :service => "riak #{type} #{percentile}",
+          :state => state,
+          :metric => val,
+          :description => "#{val} ms"
+        )
+      end
+    end
+  end
+
+  def tick
+    # This can utterly destroy a cluster, so we disable
+    # check_keys
+    check_stats
+    check_ring
+    check_disk
+  end
+end
+
+Riemann::Tools::Riak.run

data/bin/riemann-riak-keys

@@ -0,0 +1,12 @@
+#!/usr/bin/env escript
+%%! -name riakstatuscheck@127.0.0.1 -hidden
+
+main([]) -> main(["riak@127.0.0.1"]);
+main([Node]) ->
+  io:format("~w\n", [
+    lists:foldl(
+      fun({_VNode, Count}, Sum) -> Sum + Count end,
+      0,
+      rpc:call(list_to_atom(Node), riak_kv_bitcask_backend, key_counts, [])
+    )
+  ]).

data/bin/riemann-riak-ring

@@ -0,0 +1,8 @@
+#!/usr/bin/env escript
+%%! -name riakstatuscheck@127.0.0.1 -hidden
+
+main([]) -> main(["riak@127.0.0.1"]);
+main([Node]) ->
+  io:format("~p\n", [
+    rpc:call(list_to_atom(Node), riak_kv_console, ringready, [[]])
+  ]).

data/lib/riemann/tools.rb

@@ -0,0 +1,114 @@
+module Riemann
+  module Tools
+    require 'rubygems'
+    require 'trollop'
+    require 'riemann/client'
+    require 'timeout'
+
+    def self.included(base)
+      base.instance_eval do
+        def run
+          new.run
+        end
+
+        def opt(*a)
+          a.unshift :opt
+          @opts ||= []
+          @opts << a
+        end
+
+        def options
+          p = Trollop::Parser.new
+          @opts.each do |o|
+            p.send *o
+          end
+          Trollop::with_standard_exception_handling(p) do
+            p.parse ARGV
+          end
+        end
+
+        opt :host, "Riemann host", :default => '127.0.0.1'
+        opt :port, "Riemann port", :default => 5555
+        opt :event_host, "Event hostname", :type => String
+        opt :interval, "Seconds between updates", :default => 5
+        opt :tag, "Tag to add to events", :type => String, :multi => true
+        opt :ttl, "TTL for events", :type => Integer
+        opt :attribute, "Attribute to add to the event", :type => String, :multi => true
+        opt :timeout, "Timeout (in seconds) when waiting for acknowledgements", :default => 30
+        opt :tcp, "Use TCP transport instead of UDP (improves reliability, slight overhead.", :default => true
+      end
+    end
+
+    # Returns parsed options (cached) from command line.
+    def options
+      @options ||= self.class.options
+    end
+    alias :opts :options
+
+    def attributes
+      @attributes ||= Hash[options[:attribute].map do |attr|
+        k,v = attr.split(/=/)
+        if k and v
+          [k,v]
+        end
+      end]
+    end
+
+    def report(event)
+      if options[:tag]
+        # Work around a bug with beefcake which can't take frozen strings.
+        event[:tags] = options[:tag].map(&:dup)
+      end
+
+      event[:ttl] ||= (options[:ttl] || (options[:interval] * 2))
+
+      if options[:event_host]
+        event[:host] = options[:event_host].dup
+      end
+      
+      event = event.merge(attributes)
+
+      begin
+        Timeout::timeout(options[:timeout]) do
+          riemann << event
+        end
+      rescue Timeout::Error
+        riemann.connect
+      end
+    end
+
+    def new_riemann_client
+      r = Riemann::Client.new(
+        :host => options[:host],
+        :port => options[:port]
+      )
+      if options[:tcp]
+        r.tcp
+      else
+        r
+      end
+    end
+
+    def riemann
+      @riemann ||= new_riemann_client
+    end
+    alias :r :riemann
+
+    def run
+      t0 = Time.now
+      loop do
+        begin
+          tick
+        rescue => e
+          $stderr.puts "#{e.class} #{e}\n#{e.backtrace.join "\n"}"
+        end
+
+        # Sleep.
+        sleep(options[:interval] - ((Time.now - t0) % options[:interval]))
+      end
+    end
+
+    def tick
+    end
+  end
+end