remon 0.1.0

data/lib/remon/metrics/system.rb

@@ -0,0 +1,63 @@
+require 'socket'
+require_relative '../sysinfo'
+
+module Remon
+  module Metrics
+    class System
+
+      CpuStat = Struct.new(:used, :idle, :iowait)
+
+      def memory
+        m = {}
+        IO.foreach("/proc/meminfo").first(6).each do |line|
+          split = line.split(":")
+          key = split[0].to_sym
+          value = split[1].split[0]
+          m[key] = value
+        end
+        free = m[:MemAvailable].to_i || (m[:MemFree].to_i + m[:Buffers].to_i + m[:Cached].to_i)
+        total = m[:MemTotal].to_i
+        used = (1 - (free.to_f / total)).round(4)
+      end
+
+      def loadavg
+        File.open("/proc/loadavg") { |f| f.gets(" ").to_f }
+      end
+
+      def loadavg_normalized
+        (loadavg / Sysinfo.normalized_cores).round 2
+      end
+
+      def uptime
+        File.open('/proc/uptime') { |f| f.gets(" ").to_f }
+      end
+
+      def cpu_stat
+        stat = File.open('/proc/stat', 'r') { |f| f.readline }
+        @stat_regex ||= /^cpu\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/
+        mdata = stat.match(@stat_regex)
+        if mdata
+          user, nice, system = [mdata[1], mdata[2], mdata[3]].map { |i| i.to_i }
+          cpu = CpuStat.new
+          cpu.used = user + nice + system
+          cpu.idle = mdata[4].to_i
+          cpu.iowait = mdata[5].to_i
+          return cpu
+        else
+          return nil
+        end
+      end
+
+      def cpu_usage(old_cpu, new_cpu)
+        used = new_cpu.used - old_cpu.used
+        idle = new_cpu.idle - old_cpu.idle
+        iowait = new_cpu.iowait - old_cpu.iowait
+        total = used + idle + iowait
+        used_frac = (used.to_f / total).round 4
+        iowait_frac = (iowait.to_f / total).round 4
+        [used_frac, iowait_frac]
+      end
+
+    end
+  end
+end

data/lib/remon/metrics/yum.rb

@@ -0,0 +1,20 @@
+require 'remon/helper'
+
+module Remon
+  module Metrics
+    class Yum
+      include Helper
+
+      def initialize(timeout: 240)
+        @timeout = timeout
+      end
+
+      def updates_available
+        script = File.expand_path "#{__dir__}/../scripts/yum-status"
+        out = cmd "timeout -k 60s #{@timeout}s python #{script} 2>/dev/null", return_output: true, shell: true
+        count = out.chomp.to_i
+      end
+
+    end
+  end
+end

data/lib/remon/proc_check.rb

@@ -0,0 +1,26 @@
+require_relative 'check'
+module Remon
+  class ProcCheck < Check
+
+    attr_reader :name
+    def initialize(name, pr)
+      raise "no proc passed" if not pr.is_a?(Proc)
+      @name = name
+      @pr = pr
+      @mutex = Mutex.new
+    end
+
+    def run
+      instance_eval &@pr
+    end
+
+    def check_name
+      @name
+    end
+
+    def to_s
+      "<proc:#{@name}>"
+    end
+
+  end
+end

data/lib/remon/scheduler.rb

@@ -0,0 +1,106 @@
+require_relative 'logger'
+require_relative 'error'
+
+module Remon
+  class Scheduler
+
+    include Logger
+    attr_reader :queue
+
+    def initialize(schedule, queue:, scheduler_offset: 0)
+      @schedule = schedule
+      @queue = queue
+      @scheduler_pipeline = {}
+      @scheduler_offset = scheduler_offset
+      validate_schedule
+    end
+
+    def run(n = :inf, show_progress: true)
+      logger.debug "starting scheduler"
+      ticker(n, show_progress) do |t|
+        schedule_tasks (t + @scheduler_offset)
+      end
+    end
+
+    def schedule_tasks(time)
+      task_groups = keys.select { |i| time % i[:interval] == 0}
+      task_groups.each do |tg|
+        pipeline_task_group tg, time
+      end
+      enqueue_tasks(time)
+    end
+
+    def ticker(n, show_progress = false)
+      t = Time.now.to_i
+      case n
+      when Integer
+        n.times do |i|
+          puts i if show_progress
+          yield t
+          t = t + 1
+          sleep_till t
+        end
+      else
+        loop do
+          yield t
+          t = t + 1
+          sleep_till t
+        end
+      end
+    end
+
+    private
+
+    def keys
+      @keys ||= @schedule.keys
+    end
+
+    def sleep_till(t)
+      diff = (t - Time.now.to_f)
+      sleep diff if diff > 0
+    end
+
+    def pipeline_task_group(task_group, time)
+      randomize = task_group[:randomize]
+      offset = task_group[:offset]
+
+      if randomize && (offset > 0)
+        offset = rand(0..offset)
+      end
+
+      tasks = @schedule[task_group]
+      t = time + offset
+      @scheduler_pipeline[t] ||= Set.new
+      tasks.each { |task| @scheduler_pipeline[t] << task }
+    end
+
+
+    def enqueue_tasks(time)
+      tasks = @scheduler_pipeline[time]
+      return if not tasks
+      tasks.each do |t|
+        logger.debug "scheduling #{t} for time #{time}" if logger.debug?
+        @queue << t
+      end
+      @scheduler_pipeline.delete(time)
+    end
+
+    def validate_schedule
+      if not @schedule.is_a? Hash
+        raise Error, "invalid schedule: not a hash"
+      end
+
+      @schedule.each do |task_group, tasks|
+        raise Error, "invalid task_group: #{task_group}" if not valid_task_group? task_group
+      end
+    end
+
+    def valid_task_group?(t)
+      required_keys = [:interval, :randomize, :offset]
+      return false if not t.is_a? Hash
+      return false if not t.keys.all? { |i| required_keys.include? i }
+      return true
+    end
+
+  end
+end

data/lib/remon/scripts/salt-status

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+require 'json'
+
+def salt_state(output)
+  out = JSON.parse(output)
+  states = out["local"].values
+  drifted_states = states.reject { |i| i["result"] }
+  num_ok = states.count - drifted_states.count
+  total = states.count
+  state = num_ok == total ? "ok" : "warning"
+  "#{state}:#{num_ok}:#{total}"
+end
+
+timeout = ARGV.shift || 240
+begin
+  out = `timeout -k 60s #{timeout}s salt-call state.highstate test=True --out=json --state-output=changes 2>/dev/null`
+  if $?.exitstatus == 0
+    puts salt_state(out)
+  else
+    puts "error:unknown:unknown"
+  end
+rescue
+  puts "exception:unknown:unknown"
+end

data/lib/remon/scripts/yum-status

@@ -0,0 +1,12 @@
+#!/usr/bin/python
+import yum
+import json
+
+base = yum.YumBase()
+package_list = base.doPackageLists(pkgnarrow='updates', patterns='', ignore_case=True)
+
+if package_list.updates:
+  count = len(package_list.updates)
+else:
+  count = 0
+print count

data/lib/remon/sysinfo.rb

@@ -0,0 +1,69 @@
+require 'socket'
+require_relative 'helper'
+
+module Remon
+  class Sysinfo
+    extend Helper
+
+    def self.init
+      puts "loading sysinfo"
+      nproc
+      cores
+      normalized_cores
+      instance_type
+      ips
+    end
+
+    def self.instance_type
+     @instance_type ||= begin
+        exitstatus, instance_type = safe_cmd 'curl -s -m2 169.254.169.254/latest/meta-data/instance-type', return_output: true
+        exitstatus == 0 ? instance_type : "unknown"
+      end
+    end
+
+    def self.nproc
+      @nproc ||= `nproc`.chomp.strip.to_i
+    end
+
+    def self.cores
+      @cores ||= cores_linux
+    end
+
+    def self.normalized_cores
+      @normalized_cores ||= begin
+        cores = self.cores
+        nproc = self.nproc
+        (cores == nproc) ? cores : (cores + (nproc - cores)* 30.0/100)
+      end
+    end
+
+    def self.ips
+      @ips ||= begin
+        addr_infos = Socket.ip_address_list.select do |i|
+          i.ip_address =~ /\A([0-9]{1,3}\.){3}[0-9]{1,3}\z/
+        end
+        addr_infos.reject! { |i| i.ip_address == "127.0.0.1" }
+        ips = addr_infos.map { |i| i.ip_address }
+      end
+    end
+
+    private
+
+    def self.cores_linux
+      i = 0;
+      File.read("/proc/cpuinfo").split(/\n\n/).inject({}) do |cores, p|
+        physical_id = p[/physical id\s+:\s+(\d+)/, 1]
+        core_id = p[/core id\s+:\s+(\d+)/, 1]
+        if physical_id and core_id
+        cores["#{physical_id}:#{core_id}"] = true
+        elsif physical_id
+          cores["#{physical_id}:"] = true
+        else
+          cores[i += 1] = true;
+        end
+        cores
+      end.size
+    end
+
+  end
+end

data/lib/remon/version.rb

@@ -0,0 +1,3 @@
+module Remon
+  VERSION = "0.1.0"
+end

data/remon.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'remon/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "remon"
+  spec.version       = Remon::VERSION
+  spec.authors       = ["Neeraj"]
+
+  spec.summary       = %q{Health check library for system/services like postgres/elasticsearch/redis etc}
+  spec.description   = %q{Health check library for system/services like postgres/elasticsearch}
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.13.a"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "simplecov", "~> 0.12.0"
+end

data/test_config.rb

@@ -0,0 +1,44 @@
+puts Process.pid
+require 'pp'
+
+host "test-#{Socket.gethostname}"
+
+workers 1
+process_event do |event|
+  printf "%-10s %-10s %-10s %-8s %-10s\n", Time.now, event[:host], event[:state], event[:service], event[:metric]
+end
+
+mysys = check(:system).new( opts: {cpu_warning: 45})
+google_check = check(:http).new("http://google.com")
+
+filename = "test_config.rb"
+
+file_check = proc_check "test" do
+  sleep 3
+  metric = File.exist?(filename) ? 0 : 1
+  description = metric == 0 ? "file exists" : "file doesn't exist"
+  state = state(metric, warning: 1, critical: 2)
+  event({
+    service: name,
+    description: description,
+    metric: metric,
+    state: state
+  })
+end
+
+oom_check = check(:oom).new(log_file: "spec/test_data/messages")
+
+every 1.seconds, offset: 0.seconds, randomize: true do
+  run_check mysys
+  run_check :disk
+  run_check oom_check
+end
+
+every 5.seconds, offset: 2.seconds do
+  run_check google_check
+end
+
+every 60.seconds do
+#  run_check oom_check
+#  run_check google_check
+end

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: remon
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Neeraj
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-07-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.a
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.a
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+description: Health check library for system/services like postgres/elasticsearch
+email: 
+executables:
+- remon
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- FEATURES.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- dev_exe/remon
+- exe/remon
+- lib/remon.rb
+- lib/remon/check.rb
+- lib/remon/check_dsl.rb
+- lib/remon/check_runner.rb
+- lib/remon/checks/consul.rb
+- lib/remon/checks/disk.rb
+- lib/remon/checks/http.rb
+- lib/remon/checks/oom.rb
+- lib/remon/checks/redis.rb
+- lib/remon/checks/salt.rb
+- lib/remon/checks/system.rb
+- lib/remon/checks/yum.rb
+- lib/remon/config.rb
+- lib/remon/custom_logger.rb
+- lib/remon/deduped_queue.rb
+- lib/remon/error.rb
+- lib/remon/event_processor.rb
+- lib/remon/ext/num_ext.rb
+- lib/remon/helper.rb
+- lib/remon/logger.rb
+- lib/remon/metrics/consul.rb
+- lib/remon/metrics/disk.rb
+- lib/remon/metrics/http.rb
+- lib/remon/metrics/oom.rb
+- lib/remon/metrics/salt.rb
+- lib/remon/metrics/system.rb
+- lib/remon/metrics/yum.rb
+- lib/remon/proc_check.rb
+- lib/remon/scheduler.rb
+- lib/remon/scripts/salt-status
+- lib/remon/scripts/yum-status
+- lib/remon/sysinfo.rb
+- lib/remon/version.rb
+- remon.gemspec
+- test_config.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Health check library for system/services like postgres/elasticsearch/redis
+  etc
+test_files: []