remon 0.1.0

data/lib/remon/checks/yum.rb

@@ -0,0 +1,30 @@
+require 'remon/metrics/yum'
+
+defcheck :yum do
+
+  def init
+    @yum = Metrics::Yum.new
+  end
+
+  def run
+    updates_available
+  end
+
+  private
+
+  def updates_available
+    service = "yum updates"
+    count = @yum.updates_available
+    state = count > 0 ? "warning" : "ok"
+    metric = state == "ok" ? 0 : 1
+    event({
+      service: service,
+      description: "#{count} updates available",
+      state: state,
+      metric: metric
+    })
+  rescue => e
+    logger.error "#{e.class}: #{e.message}"
+    warning_event service
+  end
+end

data/lib/remon/config.rb

@@ -0,0 +1,101 @@
+require 'set'
+require_relative 'error'
+require_relative 'ext/num_ext'
+require_relative 'check_dsl'
+require 'forwardable'
+
+module Remon
+  class Config
+    using NumExt
+
+    LOAD_PATHS = ["#{__dir__}/checks"]
+
+    extend Forwardable
+
+    def initialize(config_file: nil, config_dir: nil, load_paths: [])
+      @config_file = config_file
+      @config_dir = config_dir
+      @schedule = {}
+      @scheduler_offset = 0
+      @workers = 1
+      @task_group = { interval: 0, offset: 15, randomize: false }
+      load_paths = Set.new(load_paths).merge(LOAD_PATHS)
+      @dsl = CheckDsl.new load_paths.to_a
+    end
+
+    def config
+      @config_read ||= begin
+        read_config
+        true
+      end
+      {
+        schedule: @schedule,
+        scheduler_offset: @scheduler_offset,
+        process_proc: @process_proc,
+        workers: @workers
+      }
+    end
+
+    private
+
+    def_delegators :@dsl, :check, :defcheck, :proc_check
+
+    def host(host)
+      Remon.host = host
+    end
+
+    def every(secs, randomize: false, offset: 0, &block)
+      raise Error, "offset must be less than interval" if offset > secs
+      before = @task_group
+      @task_group = { interval: secs, offset: offset, randomize: randomize }
+      yield
+    ensure
+      @task_group = before
+    end
+
+    def scheduler_offset(offset)
+      @scheduler_offset = offset
+    end
+
+    def process_event(&block)
+      @process_proc = block
+    end
+
+    def workers(workers)
+      @workers = workers
+    end
+
+    def schedule_check(check, args = [], kwargs = {})
+      @schedule[@task_group] ||= Set.new
+      if not check.is_a? Check
+        kwargs[:ttl] ||= default_ttl(@task_group[:interval])
+        klass = self.check(check)
+        check = klass.new(*args, **kwargs)
+      end
+      @schedule[@task_group] << check
+    end
+
+    def default_ttl(interval)
+      3 * interval
+    end
+
+    def read_config
+      read_config_file @config_file if @config_file
+      if @config_dir
+        Dir.glob("#{@config_dir}/*.rb").each { |f| read_config_file f }
+      end
+    end
+
+    def read_config_file(file)
+      if not File.readable? file
+        raise Error, "config #{file} not readable"
+      end
+      instance_eval(File.read(file))
+    rescue NoMethodError => e
+      raise Error, "invalid option used in config: #{e.name}"
+    end
+
+    alias_method :run_check, :schedule_check
+
+  end
+end

data/lib/remon/custom_logger.rb

@@ -0,0 +1,6 @@
+require 'logger'
+
+module Remon
+  class CustomLogger
+  end
+end

data/lib/remon/deduped_queue.rb

@@ -0,0 +1,38 @@
+require_relative 'logger'
+
+module Remon
+  class DedupedQueue
+
+    include Logger
+
+    def initialize
+      @hash = {}
+      @mutex = Mutex.new
+      @queue = Queue.new
+    end
+
+    def enqueue(task)
+      @mutex.synchronize do
+        if @hash[task]
+          logger.debug "duplicate item #{task}" if logger.debug?
+          return
+        end
+        @hash[task] = true
+      end
+      @queue << task
+    end
+
+    def dequeue(non_block = false)
+      task = @queue.pop(non_block)
+      @mutex.synchronize do
+        @hash.delete task
+      end
+      task
+    end
+
+    alias_method :'<<', :enqueue
+    alias_method :pop, :dequeue
+    alias_method :shift, :dequeue
+
+  end
+end

data/lib/remon/error.rb

@@ -0,0 +1,4 @@
+module Remon
+  class Error < StandardError
+  end
+end

data/lib/remon/event_processor.rb

@@ -0,0 +1,33 @@
+require_relative 'logger'
+
+module Remon
+  class EventProcessor
+
+    include Logger
+    attr_reader :queue
+
+    def initialize(pr)
+      @proc = pr
+      @queue = Queue.new
+    end
+
+    def start
+      @thread ||= Thread.new do
+        logger.debug { "starting event processor" }
+        loop { process_event }
+      end
+    end
+
+    def process_event
+      event = @queue.pop
+      @proc.call event
+    rescue => e
+      logger.warn "warn error #{e.message}"
+    end
+
+    def stop
+      Thread.kill @thread if @thread
+    end
+
+  end
+end

data/lib/remon/ext/num_ext.rb

@@ -0,0 +1,23 @@
+module Remon
+  module NumExt
+    refine Integer do
+
+      def seconds
+        self
+      end
+
+      def second
+        self
+      end
+
+      def minutes
+        self * 60
+      end
+
+      def minute
+        self * 60
+      end
+
+    end
+  end
+end

data/lib/remon/helper.rb

@@ -0,0 +1,41 @@
+require_relative 'logger'
+module Remon
+  module Helper
+
+    include Logger
+
+    def cmd(command, error_msg: nil, return_output: true, env: {}, shell: false)
+      if command.is_a? Array
+        command_arr = command
+        command_str = command.join(" ")
+      else
+        command_arr = command.split
+        command_str = command
+      end
+      logger.debug command_str
+
+      run_command = shell ? command_str : command_arr
+      output = if return_output
+        IO.popen(env, run_command) { |f| f.read }
+      else
+        system(env, run_command, 2 => 1)
+      end
+      exitstatus = $?.exitstatus
+
+      if exitstatus != 0
+        error_msg ||= "non zero exit for \"#{command_str}\""
+        raise Error, error_msg
+      end
+      return output
+    end
+
+    def safe_cmd(*args, **kwargs)
+      output = cmd(*args, **kwargs)
+      return $?.exitstatus, output
+    rescue => e
+      logger.debug e.message
+      return -1, nil
+    end
+
+  end
+end

data/lib/remon/logger.rb

@@ -0,0 +1,17 @@
+require 'logger'
+module Remon
+  module Logger
+
+    def self.logger
+      @logger ||= begin
+        l = ::Logger.new(STDOUT)
+        l.level = ::Logger::WARN
+        l
+      end
+    end
+
+    def logger
+      ::Remon::Logger.logger
+    end
+  end
+end

data/lib/remon/metrics/consul.rb

@@ -0,0 +1,32 @@
+require 'open-uri'
+require 'json'
+
+module Remon
+  module Metrics
+    class Consul
+
+      def initialize(host: "127.0.0.1", port: 8500)
+        @host = host
+        @port = port
+      end
+
+      def failed_nodes
+        nodes = {}
+        failed_serf_checks = critical_checks.select { |i| i["CheckID"] == "serfHealth" }
+        failed_serf_checks.each { |i| nodes[i["Node"]] = i["Output"] }
+        nodes
+      end
+
+      private
+
+      def consul_url(path)
+        "http://#{@host}:#{@port}#{path}"
+      end
+
+      def critical_checks
+        url = consul_url("/v1/health/state/critical")
+        JSON.parse(open(url).read)
+      end
+    end
+  end
+end

data/lib/remon/metrics/disk.rb

@@ -0,0 +1,24 @@
+module Remon
+  module Metrics
+    class Disk
+
+      def disks_usage
+        disks = []
+        IO.popen(['df', '-h']) do |io|
+          io.each_line do |l|
+            f = l.split(/\s+/)
+            next if f[0] == 'Filesystem'
+            next unless f[0] =~ /\// # Needs at least one slash in the mount path
+
+            disk_info = {}
+            disk_info[:mount] = f[5]
+            disk_info[:percent] = (f[4].to_f/100).round(2)
+            disk_info[:size] = f[1]
+            disks << disk_info
+          end
+        end
+        disks
+      end
+    end
+  end
+end

data/lib/remon/metrics/http.rb

@@ -0,0 +1,40 @@
+require 'open-uri'
+require 'benchmark'
+
+module Remon
+  module Metrics
+    class Http
+
+      def initialize(url)
+        @uri = URI.parse(url)
+      end
+
+      def status(read_timeout: 1, open_timeout: 1)
+        status = nil
+        time = Benchmark.realtime do
+          status = get_status(read_timeout, open_timeout)
+        end
+        return time, status
+      end
+
+      private
+
+      def get_status(read_timeout, open_timeout)
+        @uri.open(read_timeout: read_timeout, open_timeout: open_timeout, redirect: false) do |f|
+          f.status[0].to_i
+        end
+      rescue EOFError
+        return 444
+      rescue Errno::ECONNREFUSED
+        return 502
+      rescue Net::OpenTimeout
+        return 504
+      rescue Net::ReadTimeout
+        return 504
+      rescue OpenURI::HTTPRedirect => e
+        return 301
+      end
+
+    end
+  end
+end

data/lib/remon/metrics/oom.rb

@@ -0,0 +1,32 @@
+module Remon
+  module Metrics
+    class Oom
+
+      def initialize(log_file)
+        @log_file = log_file
+        raise Error, "#{log_file} not readable" if not File.readable? log_file
+      end
+
+      def stats
+        counts = oom_counts
+        total_count = counts.values.reduce(&:+)
+        todays_count = counts[Time.now.strftime("%b%e")]
+        {today: todays_count, total: total_count}
+      end
+
+      private
+
+      def oom_counts
+        counts = Hash.new(0)
+        IO.popen "grep 'invoked oom-killer' #{@log_file} | awk '{print $1 $2}' | uniq -c" do |f|
+          f.each_line do |line|
+            split = line.strip.split
+            counts[split[1]] = split[0].to_i
+          end
+        end
+        counts
+      end
+
+    end
+  end
+end

data/lib/remon/metrics/salt.rb

@@ -0,0 +1,18 @@
+module Remon
+  module Metrics
+    class Salt
+
+      def initialize(timeout: 240)
+        @timeout = timeout
+      end
+
+      def status
+        script = File.expand_path("#{__dir__}/../scripts/salt-status")
+        out = `ruby #{script} #{@timeout} 2>/dev/null`
+        output = out.chomp.split(":")
+        { state: output[0], ok: output[1].to_i, total: output[2].to_i }
+      end
+
+    end
+  end
+end