riemann-tools 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5b58978e25c07331c031e4575ffb46b7a04c3db5
-  data.tar.gz: 50649c44cd74a42dd1b7d9b82e27cbdfdb16341d
+  metadata.gz: db9b835831c44292e608818f39aded205bc4f0f9
+  data.tar.gz: 0049cb2ffc88932a2f7b9528b1760a1e571fc99c
 SHA512:
-  metadata.gz: 2d2a3f364ed87b7d79252eafc11fb5d05c5ef000cc029460c08b1b1bd4457800329e3cf69dda990458e2fb91900b02413ebcb28c677ab696bb88a46790f14ef1
-  data.tar.gz: 491e357bbf23baac6fbee6172a6f8ecb81fb004280c4135bfe67d00945785bbc190531b97a93ff06fbcd248bb5e178243a2effc92ddb5f31ad198c342fc4d39a
+  metadata.gz: 67c75f0cdba4c515c26e0d097b8b6675c2572341dbc5cb21256f512abcbe89ec4ecf28a3e9bd43d97702923e5a119016fd02dc0706e8f95a381ba95ef4b9fa43
+  data.tar.gz: 471dbf8c04d0f35fb396e900ae4d64dc6de2e4c6cc16f92a4d3b5f76870d34577cc58b6b4ac306b85cb9a576b55db2a861d76cdc0bacd4502285a9036a00a820

data/bin/riemann-freeswitch

@@ -5,26 +5,107 @@ require File.expand_path('../../lib/riemann/tools', __FILE__)
 class Riemann::Tools::FreeSWITCH
   include Riemann::Tools
 
+  opt :calls_warning, "Calls warning threshold", :default => 100
+  opt :calls_critical, "Calls critical threshold", :default => 300
+  opt :pid_file, "FreeSWITCH daemon pidfile", :type => String, :default => "/var/run/freeswitch/freeswitch.pid"
+
+  def initialize
+    @limits = {
+      :calls => {:critical => opts[:calls_critical], :warning => opts[:calls_warning]}
+    }
+  end
+
+  def dead_proc?(pid)
+    begin
+      Process.getpgid(pid)
+      false
+    rescue Errno::ESRCH
+      true
+    end
+  end
+
+  def alert(service, state, metric, description)
+    report(
+      :service => service.to_s,
+      :state => state.to_s,
+      :metric => metric.to_f,
+      :description => description
+    )
+  end
+
+  def exec_with_timeout(cmd, timeout)
+    pid = Process.spawn(cmd, {[:err,:out] => :close, :pgroup => true})
+    begin
+      Timeout.timeout(timeout) do
+        Process.waitpid(pid, 0)
+        $?.exitstatus == 0
+      end
+    rescue Timeout::Error
+      Process.kill(15, -Process.getpgid(pid))
+      puts "Killed pid: #{pid}"
+      false
+    end
+  end
+
   def tick
+    # Determine how many current calls I have according to FreeSWITCH
+    fs_calls = %x[fs_cli -x "show calls count"| grep -Po '^\\d+'].to_i
+
+    # Determine how many current channels I have according to FreeSWITCH
+    fs_channels = %x[fs_cli -x "show channels count"| grep -Po '^\\d+'].to_i
+
+    # Determine how many conferences I have according to FreeSWITCH
+    fs_conferences = %x[fs_cli -x "conference list"| grep -Pco '^Conference'].to_i
+
+    # Try to read pidfile. If it fails use Devil's dummy PID
+    begin
+      fs_pid = File.read(opts[:pid_file]).to_i
+    rescue
+      puts "Couldn't read pidfile: #{opts[:pid_file]}"
+      fs_pid = -666
+    end
+
+    # Submit calls to riemann
+    if fs_calls > @limits[:calls][:critical]
+      alert "FreeSWITCH current calls", :critical, fs_calls, "Number of calls are #{fs_calls}"
+    elsif fs_calls > @limits[:calls][:warning]
+      alert "FreeSWITCH current calls", :warning, fs_calls, "Number of calls are #{fs_calls}"
+    else
+      alert "FreeSWITCH current calls", :ok, fs_calls, "Number of calls are #{fs_calls}"
+    end
+
+    # Submit channels to riemann
+    if fs_channels > @limits[:calls][:critical]
+      alert "FreeSWITCH current channels", :critical, fs_channels, "Number of channels are #{fs_channels}"
+    elsif fs_channels > @limits[:calls][:warning]
+      alert "FreeSWITCH current channels", :warning, fs_channels, "Number of channels are #{fs_channels}"
+    else
+      alert "FreeSWITCH current channels", :ok, fs_channels, "Number of channels are #{fs_channels}"
+    end
+
+    # Submit conferences to riemann
+    if fs_conferences > @limits[:calls][:critical]
+      alert "FreeSWITCH current conferences", :critical, fs_conferences, "Number of conferences are #{fs_conferences}"
+    elsif fs_conferences > @limits[:calls][:warning]
+      alert "FreeSWITCH current conferences", :warning, fs_conferences, "Number of conferences are #{fs_conferences}"
+    else
+      alert "FreeSWITCH current conferences", :ok, fs_conferences, "Number of conferences are #{fs_conferences}"
+    end
+
+    # Submit status to riemann
+    if dead_proc?(fs_pid)
+      alert "FreeSWITCH status", :critical, -1, "FreeSWITCH service status: not running"
+    else
+      alert "FreeSWITCH status", :ok, nil, "FreeSWITCH service status: running"
+    end
+
+    # Submit CLI status to riemann using timeout in case it's unresponsive
+    if exec_with_timeout("fs_cli -x status", 2)
+      alert "FreeSWITCH CLI status", :ok, nil, "FreeSWITCH CLI status: responsive"
+    else
+      alert "FreeSWITCH CLI status", :critical, -1, "FreeSWITCH CLI status: not responding"
+    end
 
-  #determine how many current calls I have according to FreeSWITCH
-  fs_calls = %x[fs_cli -x "show calls count"| grep -Po '^\\d+']
-  
-  #determine how many current channels I have according to FreeSWITCH
-  fs_channels = %x[fs_cli -x "show channels count"| grep -Po '^\\d+']
-
-  #submit them to riemann
-  report(
-     :service => "FreeSWITCH current calls",
-     :metric => fs_calls.to_i,
-     :state => "info"
-       )
-
-  report(
-     :service => "FreeSWITCH current channels",
-     :metric => fs_channels.to_i,
-     :state => "info"
-       )
   end
 end
 

data/bin/riemann-health

@@ -114,7 +114,7 @@ class Riemann::Tools::Health
       total = used + i2-i1
       fraction = used.to_f / total
 
-      report_pct :cpu, fraction, "user+nice+sytem\n\n#{`ps -eo pcpu,pid,comm | sort -nrb -k1 | head -10`.chomp}"
+      report_pct :cpu, fraction, "user+nice+system\n\n#{`ps -eo pcpu,pid,comm | sort -nrb -k1 | head -10`.chomp}"
     end
 
     @old_cpu = [u2, n2, s2, i2]
@@ -196,6 +196,14 @@ class Riemann::Tools::Health
         used = mdat[7].to_i * (1024 ** "BKMGT".index(mdat[8]))
         free = mdat[9].to_i * (1024 ** "BKMGT".index(mdat[10]))
         @topdata[:memory] = (wired + active + used).to_f / (wired + active + used + inactive + free)
+      # This is for OSX Mavericks which
+      # uses a different format for top
+      # Example: PhysMem: 4662M used (1328M wired), 2782M unused.
+      elsif mdat = ln.match(/PhysMem: ([0-9]+)([BKMGT]) used \(([0-9]+)([BKMGT]) wired\), ([0-9]+)([BKMGT]) unused/i)
+        used  = mdat[1].to_i * (1024 ** "BKMGT".index(mdat[2]))
+        wired = mdat[3].to_i * (1024 ** "BKMGT".index(mdat[4]))
+        unused = mdat[5].to_i * (1024 ** "BKMGT".index(mdat[6]))
+        @topdata[:memory] = (used).to_f / (used + unused)
       end
     end
   end

data/bin/riemann-net

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-# Gathers munin statistics and submits them to Riemann.
+# Gathers network interface statistics and submits them to Riemann.
 
 require File.expand_path('../../lib/riemann/tools', __FILE__)
 

data/bin/riemann-rabbitmq

@@ -1,5 +1,4 @@
 #!/usr/bin/env ruby
-#
 
 require File.expand_path('../../lib/riemann/tools', __FILE__)
 
@@ -16,11 +15,28 @@ class Riemann::Tools::Rabbitmq
 
   opt :monitor_user, 'RabbitMQ monitoring user', type: :string
   opt :monitor_pass, 'RabbitMQ monitoring user password', type: :string
-  opt :monitor_port, 'RabbitMQ monitoring port', default: 15672
-  opt :monitor_host, 'RabbitMQ monitoring host', default: "localhost"
+  opt :monitor_port, 'RabbitMQ monitoring port', type: :int, default: 15672
+  opt :monitor_host, 'RabbitMQ monitoring host', type: :string, default: "localhost"
 
-  def monitor_url
-    "http://#{options[:monitor_user]}:#{options[:monitor_pass]}@#{options[:monitor_host]}:#{options[:monitor_port]}/api/overview"
+  opt :max_queue_size, "max number of items in a queue that is acceptable", type: :int, default: 1_000_000
+  opt :ignore_max_size_queues, "A regular expression to match queues that shouldn't be size-checked", type: :string
+
+  opt :node, "Specify a node to monitor", type: :strings
+
+  def base_url
+    "http://#{options[:monitor_user]}:#{options[:monitor_pass]}@#{options[:monitor_host]}:#{options[:monitor_port]}/api"
+  end
+
+  def overview_url
+    "#{base_url}/overview"
+  end
+
+  def node_url(n)
+    "#{base_url}/nodes/#{n}"
+  end
+
+  def queues_url
+    "#{base_url}/queues"
   end
 
   def event_host
@@ -40,6 +56,11 @@ class Riemann::Tools::Rabbitmq
           req.options[:timeout] = options[:read_timeout]
           req.options[:open_timeout] = options[:open_timeout]
         end
+        report(:host => event_host,
+          :service => "rabbitmq monitoring",
+          :state   => 'ok',
+          :description => "Monitoring operational"
+        )
       rescue => e
         report(:host => event_host,
           :service => "rabbitmq monitoring",
@@ -50,8 +71,92 @@ class Riemann::Tools::Rabbitmq
       response
   end
 
-  def tick
-    uri = URI(monitor_url)
+  def check_queues
+    response = safe_get(queues_url, event_host)
+    max_size_check_filter = if options[:ignore_max_size_queues]
+      Regexp.new(options[:ignore_max_size_queues])
+    else
+      nil
+    end
+
+    return if response.nil?
+
+    json = JSON.parse(response.body)
+
+    if response.status != 200
+        report(:host => event_host,
+          :service => "rabbitmq.queue",
+          :state => "critical",
+          :description => "HTTP connection error to /api/queues: #{response.status} - #{response.body}"
+        )
+    else
+      report(:host => event_host,
+        :service => "rabbitmq.queue",
+        :state => "ok",
+        :description => "HTTP connection ok"
+      )
+
+      json = JSON.parse(response.body)
+
+      json.each do |queue|
+        svc = "rabbitmq.queue.#{queue['vhost']}.#{queue['name']}"
+        errs = []
+
+        if queue['messages_ready'] > 0 and queue['consumers'] == 0
+          errs << "Queue has jobs but no consumers"
+        end
+
+        if (max_size_check_filter.nil? or queue['name'] !~ max_size_check_filter) and queue['messages_ready'] > options[:max_queue_size]
+          errs << "Queue has #{queue['messages_ready']} jobs"
+        end
+
+        if errs.empty?
+          report(:host    => event_host,
+                 :service => svc,
+                 :state   => "ok",
+                 :description => "Queue is looking good"
+          )
+        else
+          report(:host    => event_host,
+                 :service => svc,
+                 :state   => "critical",
+                 :description => errs.join("; ")
+          )
+        end
+
+        stats = (queue['message_stats'] || {}).merge(
+                  'messages'                        => queue['messages'],
+                  'messages_details'                => queue['messages_details'],
+                  'messages_ready'                  => queue['messages_ready'],
+                  'messages_ready_details'          => queue['messages_ready_details'],
+                  'messages_unacknowledged'         => queue['messages_unacknowledged'],
+                  'messages_unacknowledged_details' => queue['messages_unacknowledged_details'],
+                  'consumers'                       => queue['consumers'],
+                  'memory'                          => queue['memory'],
+                )
+
+        stats.each_pair do |k,v|
+          service = "#{svc}.#{k}"
+          if k =~ /details$/
+            metric = v['rate']
+          else
+            metric = v
+          end
+
+          # TODO: Set state via thresholds which can be configured
+
+          report(:host => event_host,
+                 :service => service,
+                 :metric => metric,
+                 :description => "RabbitMQ monitor"
+          )
+        end
+      end
+    end
+  end
+
+  def check_overview
+    uri = URI(overview_url)
     response = safe_get(uri, event_host)
 
     return if response.nil?
@@ -95,5 +200,63 @@ class Riemann::Tools::Rabbitmq
       end
     end
   end
+
+  def check_node
+    opts[:node].each do |n|
+      uri = URI(node_url(n))
+      response = safe_get(uri, event_host)
+
+      return if response.nil?
+
+      if response.status != 200
+        if response.status == 404
+          report(:host => event_host,
+            :service => "rabbitmq.node.#{n}",
+            :state => "critical",
+            :description => "Node was not found in the cluster"
+          )
+        else
+          report(:host => event_host,
+            :service => "rabbitmq.node.#{n}",
+            :state => "critical",
+            :description => "HTTP error: #{response.status} - #{response.body}"
+          )
+        end
+        return
+      end
+
+      json = JSON.parse(response.body)
+
+      if json['mem_alarm']
+        report(:host => event_host,
+          :service => "rabbitmq.node.#{n}",
+          :state     => "critical",
+          :description => "Memory alarm has triggered; job submission throttled"
+        )
+        return
+      end
+
+      if json['disk_free_alarm']
+        report(:host => event_host,
+          :service => "rabbitmq.node.#{n}",
+          :state     => "critical",
+          :description => "Disk free alarm has triggered; job submission throttled"
+        )
+        return
+      end
+
+      report(:host => event_host,
+        :service => "rabbitmq.node.#{n}",
+        :state   => "ok",
+        :description => "Node looks OK to me"
+      )
+    end
+  end
+
+  def tick
+    check_overview
+    check_node if opts[:node]
+    check_queues
+  end
 end
 Riemann::Tools::Rabbitmq.run

data/bin/riemann-riak

@@ -27,8 +27,8 @@ class Riemann::Tools::Riak
 
   def initialize
     detect_features
-    
-    @httpstatus = true 
+
+    @httpstatus = true
     # What's going on here? --aphyr
     if
       begin
@@ -59,7 +59,7 @@ class Riemann::Tools::Riak
   def detect_features
     @escript = true # Whether escript is present on this machine
     @riakadmin = true # Whether riak-admin is present
-    
+
     if `which escript` =~ /^\s*$/
       @escript = false
     end
@@ -113,6 +113,34 @@ class Riemann::Tools::Riak
     end
   end
 
+  def check_transfers
+    str = if @riakadmin
+      `riak-admin transfers`
+    else
+      nil
+    end
+
+    return if str.nil?
+
+    if str =~ /'#{opts[:node_name]}' waiting to handoff (\d+) partitions/
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak transfers',
+        :state   => 'critical',
+        :metric  => $1.to_i,
+        :description => "waiting to handoff #{$1} partitions"
+      )
+    else
+      report(
+        :host => opts[:riak_host],
+        :service => 'riak transfers',
+        :state   => 'ok',
+        :metric  => 0,
+        :description => "No pending transfers"
+      )
+    end
+  end
+
   def check_disk
     gb = `du -Ls #{opts[:data_dir]}`.split(/\s+/).first.to_i / (1024.0**2)
     report(
@@ -125,8 +153,8 @@ class Riemann::Tools::Riak
   end
 
   # Returns the riak stat for the given fsm type and percentile.
-  def fsm_stat(type, percentile)
-    "node_#{type}_fsm_time_#{percentile == 50 ? 'median' : percentile}"
+  def fsm_stat(type, property, percentile)
+    "node_#{type}_fsm_#{property}_#{percentile == 50 ? 'median' : percentile}"
   end
 
   # Returns the alerts state for the given fsm.
@@ -209,11 +237,14 @@ class Riemann::Tools::Riak
      'vnode_puts',
      'node_gets',
      'node_puts',
+     'node_gets_set',
+     'node_puts_set',
      'read_repairs']
   end
 
   def fsm_types
-    ['get', 'put']
+    [{'get' => 'time'}, {'put' => 'time'},
+     {'get' => 'set_objsize'}]
   end
 
   def fsm_percentiles
@@ -233,9 +264,11 @@ class Riemann::Tools::Riak
       core_services.each do |s|
         report(event.merge(:service => "riak #{s}"))
       end
-      fsm_types.each do |type|
-        fsm_percentiles.each do |percentile|
-          report(event.merge(:service => "riak #{type} #{percentile}"))
+      fsm_types.each do |typespec|
+        typespec.each do |type, prop|
+          fsm_percentiles.each do |percentile|
+            report(event.merge(:service => "riak #{type} #{prop} #{percentile}"))
+          end
         end
       end
       return
@@ -260,19 +293,25 @@ class Riemann::Tools::Riak
     end
 
     # FSMs
-    fsm_types.each do |type|
-      fsm_percentiles.each do |percentile|
-        val = stats[fsm_stat(type, percentile)].to_i || 0
-        val = 0 if val == 'undefined'
-        val /= 1000.0 # Convert us to ms
-        state = fsm_state(type, percentile, val)
-        report(
-          :host => opts[:riak_host],
-          :service => "riak #{type} #{percentile}",
-          :state => state,
-          :metric => val,
-          :description => "#{val} ms"
-        )
+    fsm_types.each do |typespec|
+      typespec.each do |type, prop|
+        fsm_percentiles.each do |percentile|
+          val = stats[fsm_stat(type, prop, percentile)].to_i || 0
+          val = 0 if val == 'undefined'
+          val /= 1000.0 if prop == 'time' # Convert us to ms
+          if prop == 'time'
+            state = fsm_state(type,  percentile, val)
+          else
+            state = "ok"
+          end
+          report(
+            :host => opts[:riak_host],
+            :service => "riak #{type} #{prop} #{percentile}",
+            :state => state,
+            :metric => val,
+            :description => "#{val} ms"
+          )
+        end
       end
     end
   end
@@ -283,6 +322,7 @@ class Riemann::Tools::Riak
     check_stats
     check_ring
     check_disk
+    check_transfers
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: riemann-tools
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Kyle Kingsbury
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-30 00:00:00.000000000 Z
+date: 2015-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: riemann-client