riemann-tools 1.1.1 → 1.2.0

data/lib/riemann/tools/health.rb

@@ -2,7 +2,9 @@
 
 require 'riemann/tools'
 require 'riemann/tools/utils'
+require 'riemann/tools/uptime_parser.tab'
 
+# Reports current CPU, disk, load average, and memory use to riemann.
 module Riemann
   module Tools
     class Health
@@ -14,12 +16,18 @@ module Riemann
       opt :disk_warning, 'Disk warning threshold (fraction of space used)', default: 0.9
       opt :disk_critical, 'Disk critical threshold (fraction of space used)', default: 0.95
       opt :disk_ignorefs, 'A list of filesystem types to ignore',
-          default: %w[anon_inodefs autofs cd9660 devfs devtmpfs fdescfs iso9660 linprocfs linsysfs nfs overlay procfs tmpfs]
+          default: %w[anon_inodefs autofs cd9660 devfs devtmpfs fdescfs iso9660 linprocfs linsysfs nfs overlay procfs squashfs tmpfs]
       opt :load_warning, 'Load warning threshold (load average / core)', default: 3.0
       opt :load_critical, 'Load critical threshold (load average / core)', default: 8.0
       opt :memory_warning, 'Memory warning threshold (fraction of RAM)', default: 0.85
       opt :memory_critical, 'Memory critical threshold (fraction of RAM)', default: 0.95
-      opt :checks, 'A list of checks to run.', type: :strings, default: %w[cpu load memory disk]
+      opt :uptime_warning, 'Uptime warning threshold', default: 86_400
+      opt :uptime_critical, 'Uptime critical threshold', default: 3600
+      opt :users_warning, 'Users warning threshold', default: 1
+      opt :users_critical, 'Users critical threshold', default: 1
+      opt :swap_warning, 'Swap warning threshold', default: 0.4
+      opt :swap_critical, 'Swap critical threshold', default: 0.5
+      opt :checks, 'A list of checks to run.', type: :strings, default: %w[cpu load memory disk swap]
 
       def initialize
         @limits = {
@@ -27,6 +35,9 @@ module Riemann
           disk: { critical: opts[:disk_critical], warning: opts[:disk_warning] },
           load: { critical: opts[:load_critical], warning: opts[:load_warning] },
           memory: { critical: opts[:memory_critical], warning: opts[:memory_warning] },
+          uptime: { critical: opts[:uptime_critical], warning: opts[:uptime_warning] },
+          users: { critical: opts[:users_critical], warning: opts[:users_warning] },
+          swap: { critical: opts[:swap_critical], warning: opts[:swap_warning] },
         }
         case (@ostype = `uname -s`.chomp.downcase)
         when 'darwin'
@@ -35,25 +46,32 @@ module Riemann
           @disk = method :disk
           @load = method :darwin_load
           @memory = method :darwin_memory
-          darwin_top
+          @uptime = method :bsd_uptime
+          @swap = method :bsd_swap
         when 'freebsd'
           @cores = `sysctl -n hw.ncpu`.to_i
           @cpu = method :freebsd_cpu
           @disk = method :disk
           @load = method :bsd_load
           @memory = method :freebsd_memory
+          @uptime = method :bsd_uptime
+          @swap = method :bsd_swap
         when 'openbsd'
           @cores = `sysctl -n hw.ncpu`.to_i
           @cpu = method :openbsd_cpu
           @disk = method :disk
           @load = method :bsd_load
           @memory = method :openbsd_memory
+          @uptime = method :bsd_uptime
+          @swap = method :bsd_swap
         when 'sunos'
           @cores = `mpstat -a 2>/dev/null`.split[33].to_i
           @cpu = method :sunos_cpu
           @disk = method :disk
           @load = method :bsd_load
           @memory = method :sunos_memory
+          @uptime = method :bsd_uptime
+          @swap = method :bsd_swap
         else
           @cores = `nproc`.to_i
           puts "WARNING: OS '#{@ostype}' not explicitly supported. Falling back to Linux" unless @ostype == 'linux'
@@ -61,8 +79,11 @@ module Riemann
           @disk = method :disk
           @load = method :linux_load
           @memory = method :linux_memory
+          @uptime = method :linux_uptime
+          @swap = method :linux_swap
           @supports_exclude_type = `df --help 2>&1 | grep -e "--exclude-type"` != ''
         end
+        @users = method :users
 
         opts[:checks].each do |check|
           case check
@@ -74,6 +95,12 @@ module Riemann
             @cpu_enabled = true
           when 'memory'
             @memory_enabled = true
+          when 'uptime'
+            @uptime_enabled = true
+          when 'users'
+            @users_enabled = true
+          when 'swap'
+            @swap_enabled = true
           end
         end
       end
@@ -99,6 +126,30 @@ module Riemann
         end
       end
 
+      def report_int(service, value, report)
+        return unless value
+
+        if value >= @limits[service][:critical]
+          alert service, :critical, value, "#{value} #{report}"
+        elsif value >= @limits[service][:warning]
+          alert service, :warning, value, "#{value} #{report}"
+        else
+          alert service, :ok, value, "#{value} #{report}"
+        end
+      end
+
+      def report_uptime(uptime)
+        description = uptime_to_human(uptime)
+
+        if uptime < @limits[:uptime][:critical]
+          alert 'uptime', :critical, uptime, description
+        elsif uptime < @limits[:uptime][:warning]
+          alert 'uptime', :warning, uptime, description
+        else
+          alert 'uptime', :ok, uptime, description
+        end
+      end
+
       def linux_cpu
         new = File.read('/proc/stat')
         unless new[/cpu\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/]
@@ -205,9 +256,16 @@ module Riemann
         @old_cpu = [u2, s2, t2, i2]
       end
 
+      def uptime_parser
+        @uptime_parser ||= UptimeParser.new
+      end
+
+      def uptime
+        @cached_data[:uptime] ||= uptime_parser.parse(`uptime`)
+      end
+
       def bsd_load
-        m = `uptime`.split(':')[-1].chomp.gsub(/\s+/, '').split(',')
-        load = m[0].to_f / @cores
+        load = uptime[:load_averages][1] / @cores
         if load > @limits[:load][:critical]
           alert 'load', :critical, load, "1-minute load average/core is #{load}"
         elsif load > @limits[:load][:warning]
@@ -240,47 +298,50 @@ module Riemann
       end
 
       def darwin_top
+        return @cached_data[:darwin_top] if @cached_data[:darwin_top]
+
         raw = `top -l 1 | grep -i "^\\(cpu\\|physmem\\|load\\)"`.chomp
-        @topdata = { stamp: Time.now.to_i }
+        topdata = {}
         raw.each_line do |ln|
           if ln.match(/Load Avg: [0-9.]+, [0-9.]+, ([0-9.])+/i)
-            @topdata[:load] = Regexp.last_match(1).to_f
+            topdata[:load] = Regexp.last_match(1).to_f
           elsif ln.match(/CPU usage: [0-9.]+% user, [0-9.]+% sys, ([0-9.]+)% idle/i)
-            @topdata[:cpu] = 1 - (Regexp.last_match(1).to_f / 100)
+            topdata[:cpu] = 1 - (Regexp.last_match(1).to_f / 100)
           elsif (mdat = ln.match(/PhysMem: ([0-9]+)([BKMGT]) wired, ([0-9]+)([BKMGT]) active, ([0-9]+)([BKMGT]) inactive, ([0-9]+)([BKMGT]) used, ([0-9]+)([BKMGT]) free/i))
             wired = mdat[1].to_i * (1024**'BKMGT'.index(mdat[2]))
             active = mdat[3].to_i * (1024**'BKMGT'.index(mdat[4]))
             inactive = mdat[5].to_i * (1024**'BKMGT'.index(mdat[6]))
             used = mdat[7].to_i * (1024**'BKMGT'.index(mdat[8]))
             free = mdat[9].to_i * (1024**'BKMGT'.index(mdat[10]))
-            @topdata[:memory] = (wired + active + used).to_f / (wired + active + used + inactive + free)
+            topdata[:memory] = (wired + active + used).to_f / (wired + active + used + inactive + free)
           # This is for OSX Mavericks which
           # uses a different format for top
           # Example: PhysMem: 4662M used (1328M wired), 2782M unused.
           elsif (mdat = ln.match(/PhysMem: ([0-9]+)([BKMGT]) used \([0-9]+[BKMGT] wired\), ([0-9]+)([BKMGT]) unused/i))
             used = mdat[1].to_i * (1024**'BKMGT'.index(mdat[2]))
             unused = mdat[3].to_i * (1024**'BKMGT'.index(mdat[4]))
-            @topdata[:memory] = used.to_f / (used + unused)
+            topdata[:memory] = used.to_f / (used + unused)
           end
         end
+        @cached_data[:darwin_top] = topdata
       end
 
       def darwin_cpu
-        darwin_top unless (Time.now.to_i - @topdata[:stamp]) < opts[:interval]
-        unless @topdata[:cpu]
+        topdata = darwin_top
+        unless topdata[:cpu]
           alert 'cpu', :unknown, nil, 'unable to get CPU stats from top'
           return false
         end
-        report_pct :cpu,  @topdata[:cpu], "usage\n\n#{reverse_numeric_sort_with_header(`ps -eo pcpu,pid,comm`)}"
+        report_pct :cpu, topdata[:cpu], "usage\n\n#{reverse_numeric_sort_with_header(`ps -eo pcpu,pid,comm`)}"
       end
 
       def darwin_load
-        darwin_top unless (Time.now.to_i - @topdata[:stamp]) < opts[:interval]
-        unless @topdata[:load]
+        topdata = darwin_top
+        unless topdata[:load]
           alert 'load', :unknown, nil, 'unable to get load ave from top'
           return false
         end
-        metric = @topdata[:load] / @cores
+        metric = topdata[:load] / @cores
         if metric > @limits[:load][:critical]
           alert 'load', :critical, metric, "1-minute load average per core is #{metric}"
         elsif metric > @limits[:load][:warning]
@@ -291,12 +352,12 @@ module Riemann
       end
 
       def darwin_memory
-        darwin_top unless (Time.now.to_i - @topdata[:stamp]) < opts[:interval]
-        unless @topdata[:memory]
+        topdata = darwin_top
+        unless topdata[:memory]
           alert 'memory', :unknown, nil, 'unable to get memory data from top'
           return false
         end
-        report_pct :memory, @topdata[:memory], "usage\n\n#{reverse_numeric_sort_with_header(`ps -eo pmem,pid,comm`)}"
+        report_pct :memory, topdata[:memory], "usage\n\n#{reverse_numeric_sort_with_header(`ps -eo pmem,pid,comm`)}"
       end
 
       def df
@@ -336,11 +397,81 @@ module Riemann
         end
       end
 
+      def bsd_uptime
+        value = uptime[:uptime]
+
+        report_uptime(value)
+      end
+
+      def linux_uptime
+        value = File.read('/proc/uptime').split(/\s+/)[0].to_f
+
+        report_uptime(value)
+      end
+
+      def users
+        value = uptime[:users]
+
+        report_int(:users, value, "user#{'s' if value != 1}")
+      end
+
+      def bsd_swap
+        _device, blocks, used, _avail, _capacity = `swapinfo`.lines.last.split(/\s+/)
+
+        value = Float(used) / Integer(blocks)
+
+        report_pct :swap, value, 'used'
+      rescue ArgumentError
+        # Ignore
+      end
+
+      def linux_swap
+        total_size = 0.0
+        total_used = 0.0
+
+        File.read('/proc/swaps').lines.each_with_index do |line, n|
+          next if n.zero?
+
+          _filename, _type, size, used, _priority = line.split(/\s+/)
+
+          total_size += size.to_f
+          total_used += used.to_f
+        end
+
+        return if total_size.zero?
+
+        value = total_used / total_size
+
+        report_pct :swap, value, 'used'
+      end
+
+      def uptime_to_human(value)
+        seconds = value.to_i
+        days = seconds / 86_400
+        seconds %= 86_400
+        hrs = seconds / 3600
+        seconds %= 3600
+        mins = seconds / 60
+        [
+          ("#{days} day#{'s' if days > 1}" unless days.zero?),
+          format('%<hrs>2d:%<mins>02d', hrs: hrs, mins: mins),
+        ].compact.join(' ')
+      end
+
       def tick
+        invalidate_cache
+
         @cpu.call if @cpu_enabled
         @memory.call if @memory_enabled
         @disk.call if @disk_enabled
         @load.call if @load_enabled
+        @uptime.call if @uptime_enabled
+        @users.call if @users_enabled
+        @swap.call if @swap_enabled
+      end
+
+      def invalidate_cache
+        @cached_data = {}
       end
     end
   end

data/lib/riemann/tools/kvm.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+module Riemann
+  module Tools
+    class Kvm
+      include Riemann::Tools
+
+      def tick
+        # determine how many instances I have according to libvirt
+        kvm_instances = `LANG=C virsh list | grep -c running`
+
+        # submit them to riemann
+        report(
+          service: 'KVM Running VMs',
+          metric: kvm_instances.to_i,
+          state: 'info',
+        )
+      end
+    end
+  end
+end

data/lib/riemann/tools/memcached.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+# Gathers memcached STATS and submits them to Riemann.
+module Riemann
+  module Tools
+    class Memcached
+      include Riemann::Tools
+      require 'socket'
+
+      opt :memcached_host, 'Memcached hostname', default: 'localhost'
+      opt :memcached_port, 'Memcached port', default: 11_211
+
+      def tick
+        sock = TCPSocket.new(opts[:memcached_host], opts[:memcached_port])
+        sock.print("stats\r\n")
+        sock.flush
+        stats = sock.gets
+
+        loop do
+          stats = sock.gets
+          break if stats.strip == 'END'
+
+          m = stats.match(/STAT (\w+) (\S+)/)
+          report(
+            host: opts[:memcached_host].dup,
+            service: "memcached #{m[1]}",
+            metric: m[2].to_f,
+            state: 'ok',
+            tags: ['memcached'],
+          )
+        end
+        sock.close
+      end
+    end
+  end
+end

data/lib/riemann/tools/net.rb

@@ -2,6 +2,7 @@
 
 require 'riemann/tools'
 
+# Gathers network interface statistics and submits them to Riemann.
 module Riemann
   module Tools
     class Net
@@ -42,7 +43,7 @@ module Riemann
            'tx bytes',
            'tx packets',
            'tx errs',
-           'tx drops',
+           'tx drop',
            'tx fifo',
            'tx colls',
            'tx carrier',

data/lib/riemann/tools/nginx_status.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+# Gathers nginx status stub statistics and submits them to Riemann.
+# See http://wiki.nginx.org/HttpStubStatusModule for configuring Nginx appropriately
+module Riemann
+  module Tools
+    class NginxStatus
+      include Riemann::Tools
+      require 'net/http'
+      require 'uri'
+
+      opt :uri, 'Nginx Stub Status URI', default: 'http://localhost:8080/nginx_status'
+      opt :checks, 'Which metrics to report.', type: :strings,
+                                               default: %w[active accepted handled requests reading writing waiting]
+      opt :active_warning, 'Active connections warning threshold', default: 0
+      opt :active_critical, 'Active connections critical threshold', default: 0
+      opt :reading_warning, 'Reading connections warning threshold', default: 0
+      opt :reading_critical, 'Reading connections critical threshold', default: 0
+      opt :writing_warning, 'Writing connections warning threshold', default: 0
+      opt :writing_critical, 'Writing connections critical threshold', default: 0
+      opt :waiting_warning, 'Waiting connections warning threshold', default: 0
+      opt :waiting_critical, 'Waiting connections critical threshold', default: 0
+
+      def initialize
+        @uri = URI.parse(opts[:uri])
+
+        # sample response:
+        #
+        # Active connections: 1
+        # server accepts handled requests
+        #  39 39 39
+        # Reading: 0 Writing: 1 Waiting: 0
+        @keys = %w[active accepted handled requests reading writing waiting]
+        @re = /Active connections: (\d+) \n.+\n (\d+) (\d+) (\d+) \nReading: (\d+) Writing: (\d+) Waiting: (\d+)/m
+      end
+
+      def state(key, value)
+        if opts.key? "#{key}_critical".to_sym
+          critical_threshold = opts["#{key}_critical".to_sym]
+          return 'critical' if critical_threshold.positive? && (value >= critical_threshold)
+        end
+
+        if opts.key? "#{key}_warning".to_sym
+          warning_threshold = opts["#{key}_warning".to_sym]
+          return 'warning' if warning_threshold.positive? && (value >= warning_threshold)
+        end
+
+        'ok'
+      end
+
+      def tick
+        response = nil
+        begin
+          response = ::Net::HTTP.get(@uri)
+        rescue StandardError => e
+          report(
+            service: 'nginx health',
+            state: 'critical',
+            description: "Connection error: #{e.class} - #{e.message}",
+          )
+        end
+
+        return if response.nil?
+
+        report(
+          service: 'nginx health',
+          state: 'ok',
+          description: 'Nginx status connection ok',
+        )
+
+        values = @re.match(response).to_a[1, 7].map(&:to_i)
+
+        @keys.zip(values).each do |key, value|
+          report({
+                   service: "nginx #{key}",
+                   metric: value,
+                   state: state(key, value),
+                   tags: ['nginx'],
+                 })
+        end
+      end
+    end
+  end
+end

data/lib/riemann/tools/ntp.rb

@@ -2,6 +2,7 @@
 
 require 'riemann/tools'
 
+# Reports NTP stats to Riemann.
 module Riemann
   module Tools
     class Ntp

data/lib/riemann/tools/portcheck.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+# Checks for open tcp ports.
+# (c) Max Voit 2017
+module Riemann
+  module Tools
+    class Portcheck
+      include Riemann::Tools
+      require 'socket'
+
+      opt :hostname, 'Host, defaults to localhost', default: `hostname`.chomp
+      opt :ports, "List of ports to check, e.g. '-r 80 443'", type: :ints
+
+      def initialize
+        @hostname = opts.fetch(:hostname)
+        @ports = opts.fetch(:ports)
+      end
+
+      def tick
+        @ports.each do |thisport|
+          # try opening tcp connection with 5s timeout;
+          # if this fails, the port is considered closed
+          portopen = begin
+            Socket.tcp(@hostname, thisport, connect_timeout: 5) { true }
+          rescue StandardError
+            false
+          end
+          state = if portopen
+                    'ok'
+                  else
+                    'critical'
+                  end
+          report(
+            host: @hostname.to_s,
+            service: "port #{thisport}",
+            state: state.to_s,
+            tags: ['portcheck'],
+          )
+        end
+      end
+    end
+  end
+end

data/lib/riemann/tools/proc.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'riemann/tools'
+
+# Reports running process count to riemann.
+module Riemann
+  module Tools
+    class Proc
+      include Riemann::Tools
+
+      opt :proc_regex, 'regular expression that matches the process to be monitored', type: :string, default: '.*'
+      opt :proc_min_critical, 'running process count minimum', default: 0
+      opt :proc_max_critical, 'running process count maximum', default: 65_536
+
+      def initialize
+        @limits = { critical: { min: opts[:proc_min_critical], max: opts[:proc_max_critical] } }
+
+        abort 'FATAL: specify a process regular expression, see --help for usage' unless opts[:proc_regex]
+
+        ostype = `uname -s`.chomp.downcase
+        puts "WARNING: OS '#{ostype}' not explicitly supported. Falling back to Linux" unless ostype == 'linux'
+        @check = method :linux_proc
+      end
+
+      def alert(service, state, metric, description)
+        report(
+          service: service.to_s,
+          state: state.to_s,
+          metric: metric.to_f,
+          description: description,
+        )
+      end
+
+      def linux_proc
+        process = opts[:proc_regex]
+        found = `ps axo pid=,rss=,vsize=,state=,cputime=,lstart=,command= | grep '#{process}' | grep -v grep | grep -v riemann-proc`
+        running = found.count("\n")
+        if (running > @limits[:critical][:max]) || (running < @limits[:critical][:min])
+          alert "proc count/#{process}", :critical, running, "process #{process} is running #{running} instances.\n"
+        else
+          alert "proc count/#{process}", :ok, running, "process #{process} is running #{running} instances.\n"
+        end
+        # Iterate on all the lines and create an entry for the following metrics:
+        #
+        # process/<pid>-<start-time>/rss
+        # process/<pid>-<start-time>/vsize
+        # process/<pid>-<start-time>/running
+        # process/<pid>-<start-time>/cputime
+        #
+        # description should contain the command itself.
+        # value should be either process RSS, VSIZE, or 1 if running
+        # state is always unknown for the moment
+        #
+        ps_regex = /([0-9]+) +([0-9]+) +([0-9]+) +([A-Z]) +([0-9:.]+) +[A-Za-z]{3} +([A-Za-z]{3} {1,2}[0-9]+ [0-9:]+ [0-9]+) +(.*)/
+        found.each_line do |line|
+          m = ps_regex.match(line)
+          next if m.nil?
+
+          pid, rss, vsize, state, cputime, start, command = m.captures
+          start_s = DateTime.parse(start, 'Mmm DD HH:MM:ss YYYY').to_time.to_i
+          cputime_s = DateTime.parse(cputime, '%H:%M:%S')
+          cputime_seconds = (cputime_s.hour * 3600) + (cputime_s.minute * 60) + cputime_s.second
+          running = 0
+          case state[0]
+          when 'R'
+            state_s = 'ok'
+            running = 1
+          when 'S'
+            state_s = 'ok'
+          when 'I'
+            state_s = 'warning'
+          when 'T', 'U', 'Z'
+            state_s = 'critical'
+          else
+            state_s = 'unknown'
+          end
+          report(
+            service: "proc #{pid}-#{start_s}/rss",
+            state: state_s.to_s,
+            metric: rss.to_f,
+            description: command,
+          )
+          report(
+            service: "proc #{pid}-#{start_s}/vsize",
+            state: state_s.to_s,
+            metric: vsize.to_f,
+            description: command,
+          )
+          report(
+            service: "proc #{pid}-#{start_s}/running",
+            state: state_s.to_s,
+            metric: running.to_f,
+            description: command,
+          )
+          report(
+            service: "proc #{pid}-#{start_s}/cputime",
+            state: state_s.to_s,
+            metric: cputime_seconds,
+            description: command,
+          )
+        end
+      end
+
+      def tick
+        @check.call
+      end
+    end
+  end
+end

data/lib/riemann/tools/riemann_client_wrapper.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+require 'riemann/client'
+
+module Riemann
+  module Tools
+    class RiemannClientWrapper
+      include Singleton
+
+      def initialize
+        @client = nil
+      end
+
+      def configure(options)
+        return self unless @client.nil?
+
+        r = Riemann::Client.new(
+          host: options[:host],
+          port: options[:port],
+          timeout: options[:timeout],
+          ssl: options[:tls],
+          key_file: options[:tls_key],
+          cert_file: options[:tls_cert],
+          ca_file: options[:tls_ca_cert],
+          ssl_verify: options[:tls_verify],
+        )
+
+        @client = if options[:tcp] || options[:tls]
+                    r.tcp
+                  else
+                    r
+                  end
+        self
+      end
+
+      def <<(event)
+        @client << event
+      end
+    end
+  end
+end