malevich 0.0.1

data/example/net.rb

@@ -0,0 +1,25 @@
+interval 60
+always_start true
+
+settings :include_alias, false
+settings :filter, %W('rx bytes', 'rx errs', 'rx drop', 'tx bytes', 'tx errs', 'tx drop')
+settings :words, %W('rx bytes', 'rx packets', 'rx errs', 'rx drop', 'rx fifo', 'rx frame',
+                    'rx compressed', 'rx multicast', 'tx bytes', 'tx packets', 'tx drops',
+                    'tx fifo', 'tx colls', 'tx carrier', 'tx compressed')
+
+collect "linux" do
+  File.read('/proc/net/dev').each_line do |line|
+    iface = line.split(':')[0].strip
+    iface.gsub!(/\./, '_')
+    next if (iface =~ /\./ && !settings.include_alias)
+    next unless line =~ /(\w*)\:\s*([\s\d]+)\s*/
+    settings.words.map do |service|
+      service
+    end.zip(
+      $2.split(/\s+/).map { |str| str.to_i }
+    ).each do |service, value|
+      next unless settings.filter.include? service
+      event(:service => "net #{iface} #{service}", :metric => value.to_f/interval, :diff => true)
+    end
+  end
+end

data/example/net_stat.rb

@@ -0,0 +1,25 @@
+interval 5
+settings :ports, [80, 3994]
+
+collect "linux" do
+
+  filter = nil
+  settings.ports.each do |port|
+    if filter == nil
+      filter = "\\( src *:#{port}"
+    else
+      filter += " or src *:#{port}"
+    end
+  end
+  filter += " \\) and not dst 127.0.0.1:*"
+  cmd    = 'ss -t -4 -n state established ' + filter + ' | wc -l'
+
+  count = shell!(cmd).to_i - 1
+
+  event(
+    :service     => "netstat tcp #{settings.ports.join(', ')}",
+    :metric      => count,
+    :description => "count established connects: #{count} to ports #{settings.ports.join(', ')}"
+  )
+
+end

data/example/nginx.rb

@@ -0,0 +1,22 @@
+always_start true
+interval 60
+
+settings :file, '/etc/nginx/sites-enabled/status'
+settings :url, 'http://127.0.0.1:11311/status'
+settings :nginx_status_1, %W(accepts handled requests)
+settings :nginx_status_2, %W(reading writing waiting)
+
+run_if "linux", "mac_os_x" do
+  File.exists? settings.file
+end
+
+collect "linux", "mac_os_x" do
+  lines = http_get(settings.url).split("\n")
+  lines[2].scan(/\d+/).each_with_index do |value, index|
+    event(:service => "nginx #{settings.nginx_status_1[index]}", :metric => value.to_f/interval, :diff => true)
+  end
+  event(:service => 'nginx active', :metric => lines[0].split(':')[1].strip.to_i)
+  lines[3].scan(/\d+/).each_with_index do |value, index|
+    event(:service => plugin.service + " #{settings.nginx_status_2[index]}", :metric => value.to_i)
+  end
+end

data/example/ntp.rb

@@ -0,0 +1,15 @@
+require 'net/ntp'
+
+interval 60
+warning 5
+critical 10
+settings :host, 'pool.ntp.org'
+settings :timeout, 30
+
+collect "linux", "mac_os_x" do
+  event(
+    :service     => "ntp #{settings.host}",
+    :description => "Ntp lag with host #{settings.host}",
+    :metric      => (::Net::NTP.get(plugin.host, 'ntp', settings.timeout).time.to_f - Time.now.to_f).abs
+  )
+end

data/example/pgsql.rb

@@ -0,0 +1,71 @@
+interval 60
+
+warning 120 # replication lag
+critical 500 # replication lag
+settings :host, '127.0.0.1'
+settings :user, 'postgres'
+settings :pgsql, '/usr/bin/psql'
+settings :db4monit, 'riemann_monit'
+settings :conn_warn, 5 # reserved pool connections
+settings :conn_crit, 3 # reserved pool connections
+
+run_if "linux" do
+  File.exists? settings.pgsql
+end
+
+collect "linux" do
+
+  # helpers
+  def run_sql(sql, db='postgres')
+    shell_out!("#{settings.psql} -h #{settings.host} -U #{settings.user} -tnc \"#{sql}\" #{db}").stdout
+  end
+
+  def in_recovery?
+    run_sql('select pg_is_in_recovery()') == 't'
+  end
+
+  def db4monit_exists?
+    run_sql("select 1 from pg_database where datname = '#{settings.db4monit}'") == '1'
+  end
+
+  def run_master_sql
+    run_sql("create database #{settings.db4monit}") unless db4monit_exists?
+    run_sql(
+      "drop table if exists timestamp; \
+      create table timestamp ( id int primary key, value timestamp default now() ); \
+      insert into timestamp (id) values (1); \
+          ", settings.db4monit)
+  end
+
+  def repl_lag
+    unixnow - run_sql("select extract(epoch from value::timestamp) from timestamp where id = 1;", settings.db4monit).to_i
+  end
+
+  def connections
+    max_conn = run_sql('show max_connections').to_i
+    res_conn = run_sql('show superuser_reserved_connections').to_i
+    cur_conn = run_sql('select count(1) from pg_stat_activity;').to_i
+    [cur_conn, (max_conn - res_conn - cur_conn)]
+  end
+
+  # check status
+
+  cur_conn, res_conn = connections
+  if in_recovery?
+    event(:service => 'pgsql replication lag', :description => 'Postgresql replication lag', :metric => repl_lag)
+  else
+    run_master_sql
+  end
+
+  event(:service => 'pgsql connections', :description => 'Postgresql current connections', :state => 'ok', :metric => cur_conn)
+
+  # check reserved pool size
+  if res_conn < settings.conn_warn
+    if res_conn > settings.conn_crit
+      event(:service => 'pgsql reserved connections', :description => 'Postgresql reserved connections state', :state => 'warning', :metric => res_conn)
+    else
+      event(:service => 'pgsql reserved connections', :description => 'Postgresql reserved connections state', :state => 'critical', :metric => res_conn)
+    end
+  end
+
+end

data/example/runit.rb

@@ -0,0 +1,48 @@
+interval 60
+always_start true
+
+run_if "linux" do
+  Dir.exists? '/etc/service'
+end
+
+collect "linux" do
+
+  @status_history ||= Array.new
+
+  def uptime(service)
+    pid_file = File.join(service, 'supervise', 'pid')
+    return 0 unless File.exist?(pid_file)
+    unixnow - File.mtime(pid_file).to_i
+  end
+
+  def runned?(service)
+    stat_file = File.join(service, 'supervise', 'stat')
+    return false unless File.exists?(stat_file)
+    File.read(stat_file).strip == 'run'
+  end
+
+  def human_srv(service)
+    service.gsub(/\/etc\/service\//, '')
+  end
+
+  Dir.glob('/etc/service/*').each do |srv|
+    srv_uptime = uptime(srv)
+    srv_runned = runned?(srv)
+    srv_name   = human_srv(srv)
+
+    # сервис запущен и работает дольше чем мы приходили к нему в прошлый раз
+    if srv_runned && srv_uptime > interval
+      @status_history.delete(srv_name)
+      event(:service => "runit #{srv_name}", :state => 'ok', :description => "runit service #{srv_name} running")
+    else
+      # сервис запущен но работает подозрительно мало, но последний раз замечен не был
+      if srv_uptime < interval && srv_runned && !@status_history.include?(srv_name)
+        @status_history << srv_name
+      else
+        # во всех остальных случаях сообщаем о проблеме
+        event(:service => "runit #{srv_name}", :state => 'critical', :description => "runit service #{srv_name} not running", :metric => srv_uptime)
+      end
+    end
+  end
+
+end

data/example/status_file.rb

@@ -0,0 +1,17 @@
+interval 60
+
+settings :file, '/var/tmp/error.txt'
+settings :max_lines, 100
+settings :report_lines, 5
+settings :service, 'check state file'
+
+critical 1
+
+collect "linux" do
+  content = File.read(settings.file).split("\n").delete_if { |x| x.strip.empty? }
+  event(
+    :service     => "#{settings.service} #{settings.file}",
+    :description => content.last(settings.report_lines).join("\n"),
+    :metric      => content.count
+  )
+end

data/example/tw_cli.rb

@@ -0,0 +1,17 @@
+interval 180
+always_start true
+
+settings :cmd, "/usr/sbin/tw_cli show | awk '/^c/{print $1}' | xargs -rI{} /usr/sbin/tw_cli /{} show | awk '/^[upb]/&&!/[ \t](OK|VERIFYING|VERIFY-PAUSED)/' |wc -l"
+critical 1
+
+run_if "linux" do
+  File.exists? '/usr/sbin/tw_cli'
+end
+
+collect "linux" do
+  event(
+    :service     => 'twcli',
+    :metric      => shell!(settings.cmd).to_i,
+    :description => 'Hardware raid tw_cli status'
+  )
+end

data/lib/malevich.rb

@@ -0,0 +1,59 @@
+require 'malevich/init'
+
+module Malevich
+  class Kernel
+
+    include Malevich::Loader
+
+    attr_reader :config, :cmd, :plugins, :ohai, :monitor, :events, :dsl
+
+    def initialize
+      @config  = Hashie::Mash.new # for config from file
+      @cmd     = Hashie::Mash.new
+      @plugins = Hashie::Mash.new # for plugin containers (errors, settings)
+      @logger  = Logger.new(STDOUT)
+      @monitor = Malevich::Monitor.new
+      @events  = ::Queue.new
+    end
+
+    def test_plugin(file)
+      return unless malevich.cmd[:test_given]
+      Malevich::DSL.test(file).run!
+      exit 0
+    end
+
+    def log(level = :info, message)
+      @logger.send(level.to_sym, message)
+    end
+
+    def log_level=(level = 'INFO')
+      @logger.level = Logger.const_get(level)
+    end
+
+    def ohai
+      if Time.now.to_i - @ohai_timestamp.to_i > 60 * 60
+        Kernel::log :info, 'Load ohai data'
+        real_ohai = Ohai::System.new
+        real_ohai.all_plugins
+        @ohai_timestamp = Time.now
+        @ohai           = real_ohai.data
+      else
+        @ohai
+      end
+    end
+
+  end
+end
+
+module Kernel
+
+  def malevich
+    $__malevich_utils ||= Malevich::Kernel.new
+  end
+
+  def log(level = :info, message)
+    speaker = self.class == Class ? "C: #{self.to_s}" : "I: #{self.class.to_s}"
+    malevich.log(level, "[#{speaker}] #{message}")
+  end
+
+end

data/lib/malevich/dsl.rb

@@ -0,0 +1,78 @@
+module Malevich
+  class DSL
+
+    PLUGIN_EXT = '.rb'
+
+    attr_reader :plugins
+
+    def initialize
+      @plugins = Array.new
+    end
+
+    def interval(val)
+      plugins.last.interval = val.to_f
+    end
+
+    def always_start(val)
+      plugins.last.always_start = !!val
+    end
+    alias :auto :always_start
+    alias :auto_start :always_start
+
+    def run_if(*name, &block)
+      return unless plugins.last.suitable_platform?(name)
+      plugins.last.run_if = block
+    end
+
+    def state(attr, val)
+      plugins.last.settings[attr] = val
+    end
+    alias :states :state
+
+    def critical(val)
+      plugins.last.settings[:critical] = val
+    end
+    alias :critical= :critical
+
+    def warning(val)
+      plugins.last.settings[:warning] = val
+    end
+    alias :warning= :warning
+
+    def settings(attr, val)
+      plugins.last.settings[attr] = val
+    end
+    alias :config :settings
+    alias :defaults :settings
+    alias :plugin :settings
+
+    def collect(*name, &block)
+      return unless plugins.last.suitable_platform?(name)
+      plugins.last.collect = block
+    end
+
+    def self.load(paths)
+      dsl = Malevich::DSL.new
+      paths.map do |path|
+        File.directory?(path) ? Dir["#{path}/*#{PLUGIN_EXT}"].sort : path
+      end.flatten.each do |path|
+        begin
+          log :debug, "Load plugin #{path}"
+          dsl.plugins << Malevich::Plugin.new(File.basename(path, PLUGIN_EXT))
+          dsl.instance_eval(File.read(path), path)
+        rescue ScriptError
+          dsl.plugins.pop # todo: plugin.new creates
+          log :error, "ScriptError: plugin from file #{path}"
+        end
+      end
+      dsl.plugins
+    end
+
+    def self.test(file)
+      dsl = Malevich::DSL.new
+      dsl.instance_eval(File.read(file), file)
+      dsl.plugins[0]
+    end
+
+  end
+end

data/lib/malevich/init.rb

@@ -0,0 +1,17 @@
+require 'json'
+require 'yaml'
+require 'find'
+require 'hashie'
+require 'ohai'
+require 'logger'
+require 'thread'
+require 'socket'
+require 'resolv'
+
+require 'malevich/version'
+require 'malevich/dsl'
+require 'malevich/plugin'
+require 'malevich/monitor'
+require 'malevich/loader'
+require 'malevich/responders/init'
+

data/lib/malevich/loader.rb

@@ -0,0 +1,80 @@
+module Malevich
+  module Loader
+
+    def load_responders
+      Malevich::Responder.constants.select  do |c|
+        next unless Class === Malevich::Responder.const_get(c)
+        monitor << Malevich::Responder.const_get(c).new
+      end
+    end
+
+    def load_plugins(plugin_path, config_file)
+
+      def find_plugin(name)
+        @all_plugins.find {|p| p.name == name}
+      end
+
+      @all_plugins = Malevich::DSL.load(plugin_path)
+      @all_names = Array.new
+      @plugins_to_run = Array.new
+
+      self.config.deep_merge! YAML.load_file(config_file) #rescue {}
+      self.config.each do |name, val|
+        @all_names << name
+        next if val.nil?
+        # dup plugins for val - array
+        if val.kind_of?(Array)
+          parent = find_plugin(name)
+          if parent.nil?
+            Kernel::log :error, "Unable to find parent plugin for #{name}"
+            next
+          end
+          val.each_with_index do |p_settings, i| 
+            child = parent.dup
+            child.name = "#{name}_#{i}"
+            child.settings.deep_merge!(parent.settings.dup)
+            child.settings.deep_merge!(p_settings)
+            @all_plugins << child
+            @all_names << child.name
+          end
+          # delete parent plugin
+          @all_plugins.delete(parent)
+          next 
+        end
+
+        # dup plugin with parent
+        if val.is_a?(Hash) && val.has_key?('parent')
+          parent_name = val['parent']
+          parent = find_plugin(parent_name)
+          if parent.nil?
+            Kernel::log :error, "Plugin #{parent_name} not found"
+            next
+          end
+          child = parent.dup
+          child.name = name
+          child.settings.deep_merge!(parent.settings.dup)
+          child.settings.deep_merge!(val)
+          @all_plugins << child
+          @all_names << child.name
+          next
+        end
+        # over plugins merge
+        @all_plugins.each {|p| p.settings.deep_merge!(val) if name == p.name }
+      end
+      # add plugin if it always_start or get settings and runnable
+      @all_plugins.each do |p|
+        unless p.always_start || @all_names.include?(p.name)
+          Kernel::log(:info, "Plugin '#{p.name}' not started, because it not 'always_start' and not in config")
+          next
+        end
+        @plugins_to_run << p if p.runnable?  
+      end
+      @all_plugins = nil
+      @all_names = nil
+      # start plugins! 
+      @plugins_to_run.each {|p| monitor << p }
+      @plugins_to_run = nil
+    end
+
+  end
+end