remon 0.1.0

data/lib/remon/check_dsl.rb

@@ -0,0 +1,92 @@
+require_relative 'error'
+require_relative 'check'
+require_relative 'proc_check'
+require 'set'
+
+module Remon
+  class CheckDsl
+
+    def initialize(load_paths = [])
+      @load_paths = load_paths
+      @checks = {}
+    end
+
+    def defcheck(name = nil, &block)
+      return define_klass(&block) if not name
+      name = name.to_s
+      validate_name(name)
+      if @checks[name]
+        raise Error, "check #{name} already defined"
+      end
+      klass = define_klass(&block)
+      klass.name = name
+      @checks[name] = klass
+    end
+
+    def check(name)
+      name = name.to_s
+      validate_name(name)
+      @checks[name] || load_check(name)
+    end
+
+    def proc_check(name = nil, &block)
+      ProcCheck.new(name, block)
+    end
+
+    private
+
+    def validate_name(name)
+      regex = /\A[a-zA-Z0-9_:]+\z/
+      if not name =~ regex
+        raise Error, "only alphanumeric, _, : characters allowed for check name"
+      end
+      if name.scan(/:/).size > 1
+        raise Error, "nested namespacing not allowed in check names"
+      end
+    end
+
+    def define_klass(&block)
+      Class.new(Check, &block)
+    end
+
+    def load_file(f)
+      instance_eval File.read(f), f
+    end
+
+    def load_check(name)
+      file = find_check_file(name)
+      load_file file
+      if a = @checks[name]
+        return a
+      else
+        raise Error, "unable to find check: #{name} in #{file}"
+      end
+    end
+
+    def find_check_file(name)
+      files = check_files(name)
+      combination = files.product(@load_paths).find { |f, d| Dir.glob("#{d}/#{f}").first }
+      if not combination
+        raise Error, "unable to find check: #{name} in PATH: #{@load_paths.join(":")}"
+      end
+      dir = combination[1]
+      file = combination[0]
+      path = "#{dir}/#{file}"
+    end
+
+    def check_files(name)
+      files = []
+      if name.include? ":"
+        part = name.partition(":")
+        namespace = part[0]
+        rest = part[2]
+        files << "#{namespace}/#{rest}.rb"
+        files << "#{namespace}.rb"
+      else
+        files << "#{name}.rb"
+      end
+      files
+    end
+
+  end
+end

data/lib/remon/check_runner.rb

@@ -0,0 +1,53 @@
+require_relative 'logger'
+
+module Remon
+  class CheckRunner
+    include Logger
+
+    def initialize(task_queue:, num_workers:, result_queue:)
+      @task_queue = task_queue
+      @num_workers = num_workers
+      @result_queue = result_queue
+      @workers = []
+    end
+
+    def start
+      logger.debug "starting runners"
+      @num_workers.times { @workers << new_worker }
+    end
+
+    def stop
+      @workers.each { |t| Thread.kill t if (t && t.alive?)}
+    end
+
+    def new_worker
+      Thread.new do
+        loop { process_job }
+      end
+    end
+
+    def process_job
+      j = @task_queue.pop
+      logger.debug "running #{j}" if logger.debug?
+      result = run_job j
+      case result
+      when Array
+        result.each { |r| @result_queue << r if r}
+      else
+        @result_queue << result if result
+      end
+    rescue => e
+      logger.error "error while running job #{e.message}"
+    end
+
+
+    def run_job(j)
+      if @num_workers > 1
+        j.run_mutex
+      else
+        j.run
+      end
+    end
+
+  end
+end

data/lib/remon/checks/consul.rb

@@ -0,0 +1,41 @@
+require 'remon/metrics/consul'
+
+defcheck :consul do
+
+  def init(host: "127.0.0.1", port: 8500)
+    @consul = Metrics::Consul.new(host: host, port: port)
+  end
+
+  def run
+    members_status
+  end
+
+  private
+
+  def members_status
+    failed_nodes = @consul.failed_nodes
+    state = failed_nodes.size > 0 ? "critical" : "ok"
+    event({
+      service: "consul members",
+      description: description(failed_nodes),
+      state: state,
+      metric: metric(state)
+    })
+  end
+
+  def description(failed_nodes)
+    n = failed_nodes.size
+    s = "#{n} failed nodes"
+    if n > 0
+      d = failed_nodes.map {|k,v| "#{k}: #{v}"}.join("\n")
+      "#{s}\n#{d}"
+    else
+      s
+    end
+  end
+
+  def metric(state)
+    state == "ok" ? 0 : 1
+  end
+
+end

data/lib/remon/checks/disk.rb

@@ -0,0 +1,36 @@
+require 'remon/metrics/disk'
+defcheck :disk do
+
+  opts ({
+    warning: 85,
+    critical: 95
+  })
+
+  def init
+    @disk = Metrics::Disk.new
+  end
+
+  def run
+    disk
+  end
+
+  private
+
+  def disk
+    disks = @disk.disks_usage
+    max = disks.max_by { |d| d[:percent] }
+    metric = max[:percent]
+    s = service_state(metric * 100)
+    event({
+      service: "disk",
+      description: description(disks),
+      state: s,
+      metric: metric
+    })
+  end
+
+  def description(disks)
+    disks.map { |i| "#{i[:mount]} - #{(i[:percent]*100).round 2}% #{i[:size]}"}.join("\n")
+  end
+
+end

data/lib/remon/checks/http.rb

@@ -0,0 +1,53 @@
+require 'remon/metrics/http'
+
+defcheck :http do
+
+  opts ({
+    error_statuses: [502, 500, 503],
+    read_timeout: 1,
+    open_timeout: 1
+  })
+
+  def init(url)
+    @url = url
+    @http = Metrics::Http.new(url)
+  end
+
+  def run
+    http_status
+  end
+
+  private
+
+  def http_status
+    time, status = @http.status(read_timeout: opts[:read_timeout], open_timeout: opts[:open_timeout])
+    state = state(status)
+    event({
+      service: "http #{@url}",
+      description: "#{status} in #{(time * 1000).round(2)} ms",
+      state: state,
+      metric: metric(state)
+    })
+  end
+
+  def metric(state)
+    case state
+    when "ok"
+      0
+    when "warning"
+      0.9
+    when "critical"
+      1
+    end
+  end
+
+  def state(status)
+    if status >= 500
+      "critical"
+    elsif status == 444
+      "critical"
+    else
+      "ok"
+    end
+  end
+end

data/lib/remon/checks/oom.rb

@@ -0,0 +1,26 @@
+require 'remon/metrics/oom'
+defcheck :oom do
+
+  def init(log_file: "/var/log/messages")
+    @oom = Metrics::Oom.new(log_file)
+  end
+
+  def run
+    stats = @oom.stats
+    [ oom_event(stats[:today], tag: :today),
+      oom_event(stats[:total], tag: :total)]
+  end
+
+  private
+
+  def oom_event(count, tag:)
+    state = count > 0 ? "warning" : "ok"
+    event({
+      service: "oom log #{tag}",
+      description: "#{count} times oom",
+      state: state,
+      metric: count
+    })
+  end
+
+end

data/lib/remon/checks/redis.rb

@@ -0,0 +1,23 @@
+defcheck :redis do
+
+  opts ({
+    cpu_warn: 85,
+    cpu_critical: 95
+  })
+
+  task def cpu
+    metric = rand(1..100)
+    s = state(metric, warn: o[:cpu_warn], critical: o[:cpu_critical])
+    event "cpu", state, metric, description
+    event ({
+      service: "cpu",
+      description: "cpu",
+      metric: metric,
+      state: s 
+    })
+  end
+
+#  task def memory
+  
+#  end
+end

data/lib/remon/checks/salt.rb

@@ -0,0 +1,27 @@
+require 'remon/metrics/salt'
+
+defcheck :salt do
+
+  def init
+    @salt = Metrics::Salt.new
+  end
+
+  def run
+    drift_status
+  end
+
+  private
+
+  def drift_status
+    status = @salt.status
+    state = status[:state] == "ok" ? "ok" : "warning"
+    metric = state == "ok" ? 0 : 1
+    event({
+      service: "salt",
+      description: "#{status[:state]}: #{status[:ok]}/#{status[:total]}",
+      state: state,
+      metric: metric
+    })
+  end
+
+end

data/lib/remon/checks/system.rb

@@ -0,0 +1,96 @@
+require 'remon/metrics/system'
+defcheck "system" do
+
+  opts ({
+    cpu_warning: 85,
+    cpu_critical: 95,
+
+    iowait_warning: 30,
+    iowait_critical: 70,
+
+    load_warning: 3,
+    load_critical: 8,
+
+    memory_warning: 0.85,
+    memory_critical: 0.95
+  })
+
+  def init
+    @sys = Metrics::System.new
+  end
+
+  def run
+    [*cpu_and_iowait, loadavg, memory, uptime]
+  end
+
+  def cpu_and_iowait
+    old_cpu = @old_cpu
+    new_cpu = @sys.cpu_stat
+
+    if not new_cpu
+      return e 'cpu', :unknown, nil, "/proc/stat doesn't include a CPU line"
+    end
+    @old_cpu = new_cpu
+    return nil if not old_cpu
+    used, iowait = @sys.cpu_usage(old_cpu, new_cpu)
+    [cpu_event(used), iowait_event(iowait)]
+  end
+
+  def loadavg
+    metric = @sys.loadavg_normalized
+    event({
+      service: "load",
+      metric: metric,
+      description: "1-minute load average/core is #{metric}",
+      state: service_state("load", metric)
+    })
+  end
+
+  def memory
+    metric = @sys.memory
+    description = "#{(metric * 100).round(2)}% used\n\n#{`ps -eo pmem,pid,comm | sort -nrb -k1 | head -10`.chomp}"
+
+    event({
+      service: "memory",
+      metric: metric,
+      description: description,
+      state: service_state("memory", metric)
+    })
+  end
+
+  def uptime
+    up_seconds = @sys.uptime
+    metric = (up_seconds/24/3600).round(2)
+    @ips ||= Sysinfo.ips.join(", ")
+    description = <<~HEREDOC
+      ip: "#{@ips}"
+      instance_type: "#{Sysinfo.instance_type}"
+    HEREDOC
+
+    event({
+      service: "uptime",
+      metric: metric,
+      description: description,
+      state: "ok"
+    })
+  end
+
+  private
+
+  def cpu_event(metric)
+    description = "#{(metric * 100).round(2)}% user+nice+system\n\n#{`ps -eo pcpu,pid,comm | sort -nrb -k1 | head -10`.chomp}"
+    event service: "cpu",
+          description: description,
+          metric: metric,
+          state: service_state("cpu", metric * 100)
+  end
+
+  def iowait_event(metric)
+    description = "#{metric * 100 }% iowait"
+    event service: "iowait",
+          description: description,
+          metric: metric,
+          state: service_state("iowait", metric * 100)
+  end
+
+end