kurchatov 0.0.1

data/examples/nginx_500.rb

@@ -0,0 +1,48 @@
+require 'file-tail'
+
+interval 60
+warning 10
+critical 60
+default[:check_interval] = 300
+default[:check_last_lines] = 300
+default[:file] = '/var/log/nginx/app-500.log'
+default[:nginx_time_local] = "[^\\x20]+\\x20\\+\\d{4}\\]"
+
+collect do
+
+  def get_unix_time_from_line(text)
+    match = text.match(plugin.nginx_time_local)
+    return nil unless match
+    time = parse_local_time(match[0]) rescue nil
+    time
+  end
+
+  def parse_local_time(token)
+    day, month, year, hour, minute, second, _ = token.split(/[\/: ]/) # работаем с local_time
+    Time.local(year, month, day.gsub('[', ''), hour, minute, second).to_i
+  end
+
+  count_all = 0
+  count_interval = 0
+  file = File::Tail::Logfile.new(plugin.file)
+  file.backward(plugin.check_last_lines)
+  file.readlines.each do |line|
+    line.force_encoding("UTF-8")
+    next unless line.valid_encoding?
+    time = get_unix_time_from_line(line)
+    count_interval += 1 if time > (unixnow - plugin.interval)
+    count_all += 1 if time > (unixnow - plugin.check_interval)
+  end
+
+  event(
+    :service => "nginx log parse #{plugin.file} interval errors",
+    :metric => count_all,
+    :desc => "Count errors in file #{plugin.file}, last #{plugin.check_interval} sec"
+  )
+  event(
+    :service => "nginx log parse #{plugin.file} realtime errors",
+    :metric => count_interval,
+    :state => 'ok',
+    :desc => "Count errors in file #{plugin.file}, last #{plugin.interval} sec"
+  )
+end

data/examples/ntp.rb

@@ -0,0 +1,15 @@
+require 'net/ntp'
+
+interval 60
+default[:host] = 'pool.ntp.org'
+default[:timeout] = 30
+
+collect do
+  event(
+    :service     => "ntp #{plugin.host}",
+    :desc => "Ntp lag with host #{plugin.host}",
+    :metric      => (::Net::NTP.get(plugin.host, 'ntp', plugin.timeout).time.to_f - Time.now.to_f).abs,
+    :critical => 0.5,
+    :warning => 0.1
+  )
+end

data/examples/openfiles.rb

@@ -0,0 +1,6 @@
+collect do
+  event(
+    :metric => File.read('/proc/sys/fs/file-nr').split(' ').first.to_i, 
+    :description => "Number of allocated file handles" 
+  )
+end

data/examples/pgsql.rb

@@ -0,0 +1,67 @@
+default[:host] = '127.0.0.1'
+default[:user] = 'postgres'
+default[:pgsql] = '/usr/bin/psql'
+default[:db4monit] = 'riemann_monit'
+default[:conn_warn] = 5 # reserved pool connections
+default[:conn_crit] = 3 # reserved pool connections
+
+run_if do
+  File.exists? plugin.pgsql
+end
+
+collect do
+
+  # helpers
+  def run_sql(sql, db='postgres')
+    shell_out!("#{plugin.psql} -h #{plugin.host} -U #{plugin.user} -tnc \"#{sql}\" #{db}").stdout
+  end
+
+  def in_recovery?
+    run_sql('select pg_is_in_recovery()') == 't'
+  end
+
+  def db4monit_exists?
+    run_sql("select 1 from pg_database where datname = '#{plugin.db4monit}'") == '1'
+  end
+
+  def run_master_sql
+    run_sql("create database #{plugin.db4monit}") unless db4monit_exists?
+    run_sql(
+      "drop table if exists timestamp; \
+      create table timestamp ( id int primary key, value timestamp default now() ); \
+      insert into timestamp (id) values (1); \
+          ", plugin.db4monit)
+  end
+
+  def repl_lag
+    unixnow - run_sql("select extract(epoch from value::timestamp) from timestamp where id = 1;", plugin.db4monit).to_i
+  end
+
+  def connections
+    max_conn = run_sql('show max_connections').to_i
+    res_conn = run_sql('show superuser_reserved_connections').to_i
+    cur_conn = run_sql('select count(1) from pg_stat_activity;').to_i
+    [cur_conn, (max_conn - res_conn - cur_conn)]
+  end
+
+  # check status
+
+  cur_conn, res_conn = connections
+  if in_recovery?
+    event(:service => 'pgsql replication lag', :desc => 'Postgresql replication lag', :metric => repl_lag, :warning => 120, :critical => 500)
+  else
+    run_master_sql
+  end
+
+  event(:service => 'pgsql connections', :desc => 'Postgresql current connections', :state => 'ok', :metric => cur_conn)
+
+  # check reserved pool size
+  if res_conn < plugin.conn_warn
+    if res_conn > plugin.conn_crit
+      event(:service => 'pgsql reserved connections', :desc => 'Postgresql reserved connections state', :state => 'warning', :metric => res_conn)
+    else
+      event(:service => 'pgsql reserved connections', :desc => 'Postgresql reserved connections state', :state => 'critical', :metric => res_conn)
+    end
+  end
+
+end

data/examples/ping_icmp.rb

@@ -0,0 +1,12 @@
+require 'net/ping'
+
+interval 60
+default[:host] = "localhost"
+
+collect do
+  event(
+    :service => "ping icmp #{plugin.host}",
+    :state => ::Net::Ping::External.new(ip).ping,
+    :description => "ping icmp host: #{plugin.host}"
+  )
+end

data/examples/ping_tcp.rb

@@ -0,0 +1,14 @@
+require 'net/ping'
+
+default[:host] = "localhost"
+default[:port] = 22
+default[:timeout] = 5
+
+collect do
+  ::Net::Ping::TCP.econnrefused = true
+  event(
+    :service => "ping tcp #{plugin.host}:#{plugin.port}",
+    :state => ::Net::Ping::TCP.new(plugin.host, plugin.port, plugin.timeout).ping,
+    :description => "Ping tcp #{plugin.host}:#{plugin.port}"
+  )
+end

data/examples/proc_mem.rb

@@ -0,0 +1,24 @@
+require 'sys/proctable'
+
+default[:pidfile] = '/var/run/service.pid'
+
+collect do
+
+  @pagesize ||= shell('/usr/bin/getconf PAGESIZE').to_i
+  master_pid = File.read(plugin.pidfile).to_i
+  mem_total = 0
+  ::Sys::ProcTable.ps do |p|
+    next unless p.pid == master_pid || p.ppid == master_pid
+    mem_total += (p.rss * @pagesize).to_f / 1024
+  end
+
+  event(
+    :diff => true,
+    :description => "RSS usage delta #{plugin.pidfile}",
+    :metric => mem_total,
+    :service => "procmem #{plugin.pidfile}",
+    :warning => 30*1024, #kibibytes
+    :critical => 90*1024
+  ) if File.stat(plugin.pidfile).mtime > interval
+
+end

data/examples/process_usage.rb

@@ -0,0 +1,15 @@
+interval 60
+
+default[:pid_file] = '/etc/service/name/supervise/pid'
+
+collect do
+  if File.exists? plugin.pid_file
+    pid = File.read(plugin.pid_file).chomp
+    cpu_usage = shell("ps -p #{pid} S -o pcpu h").to_i
+    mem_usage = shell("ps -p #{pid} S -o rss h").to_i
+    event(:service => "process pid cpu #{plugin.pid_file}", :metric => cpu_usage, :description => "Cpu usage for process pid: #{plugin.pid_file}", :warning => 70, :critical => 90)
+    event(:service => "process pid mem #{plugin.pid_file}", :metric => mem_usage.to_f / 1024, :description => "Mem (Mb) usage for process pid: #{plugin.pid_file}", :state => 'ok')
+  else
+    event(:service => "process pid #{plugin.pid_file}", :state => 'critical', :description => "File #{plugin.pid_file} not found")
+  end
+end

data/examples/rabbitmq.rb

@@ -0,0 +1,16 @@
+default[:api_url] = 'http://admin:admin@127.0.0.1:55672/api'
+default[:vhost] = ['notexists']
+
+collect do
+  plugin.vhosts.each do |vhost|
+    vhost_uri = "#{plugin.api_url}/queues/#{CGI.escape(vhost)}"
+    JSON.parse(rest_get(vhost_uri)).each do |queue|
+      event(
+        :service => "rabbitmq queue #{queue['name'].gsub('.', '_')} messages count",
+        :metric => queue['messages'].to_i,
+        :desc => "Rabbitmq queue count in #{queue['name']}",
+        :critical => 1000
+      )
+    end
+  end
+end

data/examples/runit.rb

@@ -0,0 +1,47 @@
+always_start true
+
+run_if do
+  Dir.exists? '/etc/service'
+end
+
+collect do
+
+  @status_history ||= Array.new
+
+  def uptime(service)
+    pid_file = File.join(service, 'supervise', 'pid')
+    return 0 unless File.exist?(pid_file)
+    unixnow - File.mtime(pid_file).to_i
+  end
+
+  def runned?(service)
+    stat_file = File.join(service, 'supervise', 'stat')
+    return false unless File.exists?(stat_file)
+    File.read(stat_file).strip == 'run'
+  end
+
+  def human_srv(service)
+    service.gsub(/\/etc\/service\//, '')
+  end
+
+  Dir.glob('/etc/service/*').each do |srv|
+    srv_uptime = uptime(srv)
+    srv_runned = runned?(srv)
+    srv_name   = human_srv(srv)
+
+    # сервис запущен и работает дольше чем мы приходили к нему в прошлый раз
+    if srv_runned && srv_uptime > interval
+      @status_history.delete(srv_name)
+      event(:service => "runit #{srv_name}", :state => 'ok', :description => "runit service #{srv_name} running")
+    else
+      # сервис запущен но работает подозрительно мало, но последний раз замечен не был
+      if srv_uptime < interval && srv_runned && !@status_history.include?(srv_name)
+        @status_history << srv_name
+      else
+        # во всех остальных случаях сообщаем о проблеме
+        event(:service => "runit #{srv_name}", :state => 'critical', :description => "runit service #{srv_name} not running", :metric => srv_uptime)
+      end
+    end
+  end
+
+end

data/examples/sidekiq.rb

@@ -0,0 +1,21 @@
+default[:url] = 'http://localhost/admin/sidekiq/dashboard/stats'
+
+collect do
+  stats = JSON.parse(rest_get(plugin.url))
+  stats = stats["sidekiq"] ? stats["sidekiq"] : stats
+  event(
+    :service => "#{name} #{plugin.url}",
+    :metric => stats["enqueued"].to_i,
+    :description => "sidekiq queue from #{plugin.url}",
+    :warning => 10,
+    :critical => 60
+  )
+  event(
+    :service => "#{name} #{plugin.url}",
+    :metric => stats["failed"].to_i,
+    :diff => true,
+    :description => "sidekiq failed from #{plugin.url}",
+    :warning => 10,
+    :critical => 60
+  )
+end

data/examples/sidekiq_queue_state.rb

@@ -0,0 +1,9 @@
+default[:url] = 'http://localhost/sidekiq/queue-state'
+
+collect do
+  event(
+    :service => "#{name} #{plugin.url}",
+    :metric => rest_get(plugin.url).strip == "OK",
+    :description => "sidekiq queue status #{plugin.url}"
+  )
+end

data/examples/status_file.rb

@@ -0,0 +1,14 @@
+default[:file] = '/var/tmp/error.txt'
+default[:max_lines] = 100
+default[:report_lines] = 5
+default[:service] = 'check state file'
+
+collect do
+  content = File.read(plugin.file).split("\n").delete_if { |x| x.strip.empty? }
+  event(
+    :service     => "#{plugin.service} #{plugin.file}",
+    :description => content.last(plugin.report_lines).join("\n"),
+    :metric      => content.count,
+    :critical    => 1
+  )
+end

data/examples/tw_cli.rb

@@ -0,0 +1,17 @@
+interval 180
+always_start true
+
+default[:cmd] = "/usr/sbin/tw_cli show | awk '/^c/{print $1}' | xargs -rI{} /usr/sbin/tw_cli /{} show | awk '/^[upb]/&&!/[ \t](OK|VERIFYING|VERIFY-PAUSED)/' |wc -l"
+
+run_if do
+  File.exists? '/usr/sbin/tw_cli'
+end
+
+collect do
+  event(
+    :service     => 'twcli',
+    :metric      => shell(plugin.cmd).to_i,
+    :description => 'Hardware raid tw_cli status',
+    :critical    => 1 
+  )
+end

data/examples/uptime.rb

@@ -0,0 +1,14 @@
+interval 60
+always_start true
+
+collect :os => "linux" do
+  event(
+    :metric => File.read('/proc/uptime').split(' ').first.to_f
+  )
+end
+
+collect :os => "darwin" do
+  event(
+    :metric => shell('uptime | cut -d":" -f4- | sed s/,//g').to_f
+  )
+end

data/kurchatov.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kurchatov/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "kurchatov"
+  spec.version       = Kurchatov::VERSION
+  spec.authors       = ["Vasiliev Dmitry"]
+  spec.email         = ["vadv.mkn@gmail.com"]
+  spec.summary       = %q{Gem for monitoring with riemann.}
+  spec.description   = %q{Gem for monitoring with riemann.}
+  spec.homepage      = "https://github.com/vadv/kurchatov"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "beefcake", ">= 0.3.5"
+  spec.add_dependency "ohai", ">= 6.20.0"
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "ruby-prof"
+end

data/lib/kurchatov/application.rb

@@ -0,0 +1,154 @@
+# encoding: utf-8
+
+require "mixlib/cli"
+require "kurchatov/version"
+require "kurchatov/config"
+require "kurchatov/mixin/init"
+require "kurchatov/log"
+require "kurchatov/plugin/config"
+require "kurchatov/monitor"
+require "kurchatov/responders/init"
+
+module Kurchatov
+  class Aplication
+    include Mixlib::CLI
+    include Kurchatov::Mixin::Ohai
+
+    option :help,
+      :short        => "-h",
+      :long         => "--help",
+      :description  => "Show this message",
+      :on           => :tail,
+      :boolean      => true,
+      :show_options => true,
+      :exit         => 1
+
+    option :version,
+      :short        => "-v",
+      :long         => "--version",
+      :description  => "Show version",
+      :boolean      => true,
+      :proc         => lambda {|v| puts "Kurchatov: #{Kurchatov::VERSION}"},
+      :exit         => 0
+
+    option :log_level,
+      :short        => "-l LEVEL",
+      :long         => "--log_level LEVEL",
+      :description  => "Set the log level (debug, info, warn, error, fatal)",
+      :proc         => lambda {|l| l.to_sym}
+
+    option :log_location,
+      :short        => "-L LOGLOCATION",
+      :long         => "--logfile LOGLOCATION",
+      :description  => "Set the log file location"
+
+    option :test_plugin,
+      :short        => "-T FILE",
+      :long         => "--test-plugin FILE",
+      :description  => "Test plugin"
+
+    option :config_file,
+      :short        => "-c FILE",
+      :long         => "--config FILE",
+      :description  => "Config file"
+
+    option :plugin_paths,
+      :short        => "-d DIR1,DIR2",
+      :long         => "--plugins DIR1,DIR2",
+      :description  => "Plugin directories",
+      :proc         => lambda {|l| l.split(',') }
+
+    option :ohai_plugins_paths,
+      :short        => "-o DIR1,DIR2",
+      :long         => "--ohai--plugins DIR1,DIR2",
+      :description  => "Plugin directories",
+      :proc         => lambda {|l| l.split(',') }
+
+    option :host,
+      :long         => "--host HOSTNAME",
+      :description  => "Set hostname for events",
+      :proc         => lambda {|l| l.split(',') }
+
+    option :tags,
+      :short        => "-t tag1,tag2,tag3",
+      :long         => "--tags tag1,tag2,tag3",
+      :description  => "Set tags for events",
+      :proc         => lambda {|l| l.split(',') }
+
+    option :riemann_responder,
+      :short        => "-H HOST1,HOST2:55655",
+      :long         => "--hosts HOST1,HOST2:55655",
+      :description  => "Set riemann hosts for send events",
+      :proc         => lambda {|l| l.split(',') }
+
+    option :http_responder,
+      :long         => "--http 0.0.0.0:55755",
+      :description  => "Set http responder for information"
+
+    option :udp_responder,
+      :long         => "--udp 0.0.0.0:55955",
+      :description  => "Set udp responder for information"
+
+    option :stop_on_error,
+      :long         => "--stop-on-error",
+      :description  => "Stop on plugin or connection problem",
+      :boolean      => true,
+      :proc         => lambda {|l| !!l }
+
+    def configure_opts
+      @attributes = parse_options
+      @attributes = nil if @attributes.empty?
+      Config.merge!(config)
+    end
+
+    def configure_logging
+      Log.init(Config[:log_location])
+      Log.level = Config[:log_level]
+    end
+
+    def configure_defaults
+      ::Ohai::Config[:plugin_path] = [ File.expand_path(File.join('..', 'ohai', 'plugins'), File.dirname(__FILE__)) ]
+      if Config[:ohai_plugins_paths]
+        ::Ohai::Config[:plugin_path] += Config[:ohai_plugins_paths]
+      end
+      Config[:host] ||= ohai[:fqdn] || ohai[:hostname]
+    end
+
+    def configure_responders
+      return if Config[:test_plugin]
+      Log.error("Please set riemann host") and exit Config[:ERROR_CONFIG] unless 
+        Config[:riemann_responder]
+      if Config[:udp_responder]
+        @monitor << Responders::Udp.new( Config[:udp_responder] )
+      end
+      if Config[:http_responder]
+        @monitor << Responders::Http.new( Config[:http_responder] )
+      end
+      @monitor << Responders::Riemann.new( Config[:riemann_responder] )
+    end 
+
+    def configure_plugins
+      return if Config[:test_plugin]
+      plugins = Kurchatov::Plugins::Config.load_plugins(Config[:plugin_paths], 
+                                                        Config[:config_file])
+      plugins.each {|p| @monitor << p }
+    end
+
+    def configure_test_plugin
+      return if !Config[:test_plugin]
+      @monitor << Kurchatov::Plugins::DSL.load_riemann_plugin(Config[:test_plugin])
+    end
+
+    def run
+      configure_opts
+      configure_logging
+      configure_defaults
+      @monitor = Monitor.new(Config[:stop_on_error] || !!Config[:test_plugin])
+      configure_responders
+      configure_plugins
+      configure_test_plugin
+      @monitor.run
+    end
+    
+  end
+end

data/lib/kurchatov/config.rb

@@ -0,0 +1,14 @@
+require "mixlib/config"
+
+module Kurchatov
+  class Config
+    extend Mixlib::Config
+    default :log_level, :info
+    default :log_location, STDERR
+    default :plugin_paths, ['/usr/share/kurchatov/plugins']
+    default :config_file, '/etc/kurchatov/config.yml'
+    # errors
+    default :ERROR_CONFIG, 2
+    default :ERROR_PLUGIN_REQ, 3
+  end
+end

data/lib/kurchatov/log.rb

@@ -0,0 +1,9 @@
+require "mixlib/log"
+
+module Kurchatov
+  class Log
+    extend Mixlib::Log
+    init(Config[:log_location])
+    level = Config[:log_level]
+  end
+end