kurchatov 0.0.1

data/lib/kurchatov/mashie.rb

@@ -0,0 +1,152 @@
+class Mashie < Hash
+
+  def initialize(source_hash = nil, default = nil, &blk)
+    deep_update(source_hash) if source_hash
+    default ? super(default) : super(&blk)
+  end
+
+  class << self; alias [] new; end
+
+  def id #:nodoc:
+    self["id"]
+  end
+
+  def type #:nodoc:
+    self["type"]
+  end
+
+  alias_method :regular_reader, :[]
+  alias_method :regular_writer, :[]=
+
+  def custom_reader(key)
+    value = regular_reader(convert_key(key))
+    yield value if block_given?
+    value
+  end
+
+  def custom_writer(key,value) #:nodoc:
+    regular_writer(convert_key(key), convert_value(value))
+  end
+
+  alias_method :[], :custom_reader
+  alias_method :[]=, :custom_writer
+
+  def initializing_reader(key)
+    ck = convert_key(key)
+    regular_writer(ck, self.class.new) unless key?(ck)
+    regular_reader(ck)
+  end
+
+  def underbang_reader(key)
+    ck = convert_key(key)
+    if key?(ck)
+      regular_reader(ck)
+    else
+      self.class.new
+    end
+  end
+
+  def fetch(key, *args)
+    super(convert_key(key), *args)
+  end
+
+  def delete(key)
+    super(convert_key(key))
+  end
+
+  alias_method :regular_dup, :dup
+  # Duplicates the current mash as a new mash.
+  def dup
+    self.class.new(self, self.default)
+  end
+
+  def key?(key)
+    super(convert_key(key))
+  end
+  alias_method :has_key?, :key?
+  alias_method :include?, :key?
+  alias_method :member?, :key?
+
+  def deep_merge(other_hash, &blk)
+    dup.deep_update(other_hash, &blk)
+  end
+  alias_method :merge, :deep_merge
+
+  def deep_update(other_hash, &blk)
+    other_hash.each_pair do |k,v|
+      key = convert_key(k)
+      if regular_reader(key).is_a?(Mash) and v.is_a?(::Hash)
+        custom_reader(key).deep_update(v, &blk)
+      else
+        value = convert_value(v, true)
+        value = blk.call(key, self[k], value) if blk
+        custom_writer(key, value)
+      end
+    end
+    self
+  end
+  alias_method :deep_merge!, :deep_update
+  alias_method :update, :deep_update
+  alias_method :merge!, :update
+
+  def shallow_merge(other_hash)
+    dup.shallow_update(other_hash)
+  end
+
+  def shallow_update(other_hash)
+    other_hash.each_pair do |k,v|
+      regular_writer(convert_key(k), convert_value(v, true))
+    end
+    self
+  end
+
+  def replace(other_hash)
+    (keys - other_hash.keys).each { |key| delete(key) }
+    other_hash.each { |key, value| self[key] = value }
+    self
+  end
+
+  def respond_to?(method_name, include_private=false)
+    return true if key?(method_name) || method_name.to_s.slice(/[=?!_]\Z/)
+    super
+  end
+
+  def method_missing(method_name, *args, &blk)
+    return self.[](method_name, &blk) if key?(method_name)
+    match = method_name.to_s.match(/(.*?)([?=!_]?)$/)
+    case match[2]
+    when "="
+      self[match[1]] = args.first
+    when "?"
+      !!self[match[1]]
+    when "!"
+      initializing_reader(match[1])
+    when "_"
+      underbang_reader(match[1])
+    else
+      default(method_name, *args, &blk)
+    end
+  end
+
+  protected
+
+  def convert_key(key) #:nodoc:
+    key.to_s
+  end
+
+  def convert_value(val, duping=false) #:nodoc:
+    case val
+    when self.class
+      val.dup
+    when Hash
+      duping ? val.dup : val
+    when ::Hash
+      val = val.dup if duping
+      self.class.new(val)
+    when Array
+      val.collect{ |e| convert_value(e) }
+    else
+      val
+    end
+  end
+end

data/lib/kurchatov/mixin/command.rb

@@ -0,0 +1,31 @@
+require "mixlib/shellout"
+
+module Kurchatov
+  module Mixin
+    module Command
+
+      def shell_out(cmd)
+        mix = ::Mixlib::ShellOut.new(cmd)
+        mix.run_command
+        mix
+      end
+
+      def shell_out!(cmd)
+        mix = shell_out(cmd)
+        mix.error!
+        mix
+      end
+
+      def shell(cmd)
+        mix = shell_out!(cmd)
+        mix.stdout
+      end
+
+      def print_unix_mem_info
+        pid, size = `ps ax -o pid,rss | grep -E "^[[:space:]]*#{$$}"`.strip.split.map(&:to_i)
+        puts "Pid: #{pid}, memusage: #{size}"
+      end
+
+    end
+  end
+end

data/lib/kurchatov/mixin/event.rb

@@ -0,0 +1,63 @@
+module Kurchatov
+  module Mixin
+    module Event
+
+      EVENT_FIELDS = [
+        :time, :state, :service, :host,
+        :description, :tags, :ttl, :metric
+      ]
+
+      def event(hash = {})
+        normilize_event(hash)
+        Log.info("Mock message for test plugin: #{hash.inspect}") if Kurchatov::Config[:test_plugin]
+        events << hash
+      end
+
+      protected
+
+      def normilize_event(hash = {})
+        hash[:description] = hash[:desc] if hash[:description].nil? && hash[:desc]
+        if hash[:metric].kind_of?(Float)
+          hash[:metric] = 0.0 if hash[:metric].nan?
+          hash[:metric] = hash[:metric].round(2)
+        end
+        set_diff_metric(hash)
+        set_event_state(hash)
+        hash.each {|k,_| hash.delete(k) unless EVENT_FIELDS.include?(k)}
+        hash[:service] ||= name
+        hash[:tags] ||= Kurchatov::Config[:tags]
+        hash[:host] ||= Kurchatov::Config[:host]
+      end
+
+      def set_event_state(hash = {})
+        return if hash[:state]
+        return if hash[:critical].nil? && hash[:warning].nil?
+        return if hash[:metric].nil?
+        if hash[:state] == true || hash[:state] == false
+          hash[:state] = hash[:state] ? 'ok' : 'critical'
+          return
+        end
+        hash[:state] = 'ok'
+        hash[:state] = 'warning' if hash[:warning] && hash[:metric] >= hash[:warning]
+        hash[:state] = 'critical' if hash[:critical] && hash[:metric] >= hash[:critical]
+      end
+
+      def set_diff_metric(hash ={})
+        hash[:diff] ||= hash[:as_diff] if hash[:as_diff]
+        return if hash[:diff].nil? && !hash[:diff]
+        return if hash[:metric].nil?
+        return if hash[:service].nil?
+        @history ||= {}
+        if @history[hash[:service]]
+          old_metric = @history[hash[:service]]
+          @history[hash[:service]] = hash[:metric]
+          hash[:metric] = hash[:metric] - old_metric
+        else
+          @history[hash[:service]] = hash[:metric]
+          hash[:metric] = nil
+        end
+      end
+
+    end
+  end
+end

data/lib/kurchatov/mixin/http.rb

@@ -0,0 +1,21 @@
+require "open-uri"
+require "yajl/json_gem"
+
+module Kurchatov
+  module Mixin
+    module Http
+
+      # /path/to/file, https://ya.ru, http://a:a@yandex.ru
+      def rest_get(url)
+        uri = URI(url)
+        if uri.userinfo
+          open("#{uri.scheme}://#{uri.hostname}:#{uri.port}#{uri.request_uri}",
+               :http_basic_authentication => [uri.user, uri.password]).read
+        else
+          open(url).read
+        end
+      end
+
+    end 
+  end 
+end

data/lib/kurchatov/mixin/init.rb

@@ -0,0 +1,6 @@
+require "ohai"
+require_relative "command"
+require_relative "http"
+require_relative "ohai"
+require_relative "queue"
+require_relative "event"

data/lib/kurchatov/mixin/ohai.rb

@@ -0,0 +1,22 @@
+module Kurchatov
+
+  module Ohai
+    def self.data
+      @ohai ||= ::Ohai::System.new
+      Log.info("Load ohai plugins")
+      @ohai.all_plugins
+      @ohai.data
+    end
+  end
+
+  module Mixin
+    module Ohai
+      class << self; attr_accessor :ohai_instance; end
+      def ohai
+        @ohai_instance ||= Kurchatov::Mixin::Ohai.ohai_instance ||= Kurchatov::Ohai.data
+      end
+    end
+  end
+
+end
+

data/lib/kurchatov/mixin/queue.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+
+require "kurchatov/queue"
+
+module Kurchatov
+  module Mixin
+    module Queue
+      class << self; attr_accessor :instance_queue end
+      def events
+        @instance_queue ||= Kurchatov::Mixin::Queue.instance_queue ||= Kurchatov::Queue.new
+      end
+    end
+  end
+end

data/lib/kurchatov/monitor.rb

@@ -0,0 +1,62 @@
+require "kurchatov/mixin/queue"
+
+module Kurchatov
+  class Monitor
+
+    class Task
+      include Kurchatov::Mixin::Event
+      include Kurchatov::Mixin::Queue
+      attr_accessor :thread, :instance
+
+      def initialize(plugin)
+        @plugin = plugin
+        @thread = Thread.new { @plugin.run }
+      end
+
+      def died?
+        return false if @thread.alive?
+        # thread died, join and extract error
+        begin
+          @thread.join # call error
+        rescue => e
+          desc = "Plugin '#{@plugin.name}' died. #{e.class}: #{e}\n #{e.backtrace.join("\n")}"
+          Log.error(desc)
+          event(:service => "riemann client errors", :desc => desc, :state => 'critical')
+        end
+        @thread = Thread.new { @plugin.run }
+        true
+      end
+
+      def status
+        {@plugin.name => @thread.alive?}
+      end
+
+    end
+
+    attr_accessor :tasks
+    CHECK_ALIVE_TIMEOUT = 5
+
+    def initialize(stop = false)
+      @stop_on_error = stop
+      @tasks = Array.new
+    end
+
+    def <<(plugin)
+      Log.debug("Add new plugin: #{plugin.inspect}")
+      @tasks << Task.new(plugin)
+    end
+
+    def run
+      loop do 
+        @tasks.each { |t| exit Config[:ERROR_PLUGIN_REQ] if t.died? && @stop_on_error }
+        Log.debug("Check alive plugins [#{@tasks.count}]")
+        sleep CHECK_ALIVE_TIMEOUT
+      end
+    end
+
+    def tasks_status
+      @tasks.map {|t| t.status }
+    end
+
+  end
+end

data/lib/kurchatov/plugin/config.rb

@@ -0,0 +1,68 @@
+require "kurchatov/plugin/riemann"
+require "kurchatov/plugin/dsl"
+require "yaml"
+
+module Kurchatov
+  module Plugins
+    module Config
+
+      def self.find_plugin(name, array)
+        array.find {|p| p.name == name }
+      end
+
+      def self.load_plugins(plugins_path, config_file)
+        Log.error("Config file #{config_file} not found") and exit Kurchatov::Config[:ERROR_CONFIG] unless File.exists?(config_file)
+        @all_plugins = Kurchatov::Plugins::DSL.load_riemann_plugins(plugins_path)
+        @all_names = Array.new
+        @plugins_to_run = Array.new
+        config = YAML.load_file(config_file)
+        config.each do |name, val|
+          @all_names << name
+          next if val.nil?
+          #
+          # dup plugins from array
+          #
+          if val.kind_of? Array
+            parent = find_plugin(name, @all_plugins)
+            Log.error("Unable to find parent plugin for #{name}") and next if parent.nil?
+            val.each_with_index do |p_settings, i|
+              child = parent.dup
+              child.name = "#{name}_#{i}"
+              child.plugin = parent.plugin.dup
+              child.plugin.merge!(p_settings)
+              @all_plugins << child
+              @all_names << child.name
+            end
+            @all_plugins.delete(parent)
+            next
+          end
+          #
+          # dup plugins from 'parent'
+          #
+          if val.is_a?(Hash) && val.has_key?('parent')
+            parent = find_plugin(val['parent'], @all_plugins)
+            Log.error("Unable to find parent '#{parent_name}' for '#{name}'") and next if parent.nil?
+            child = parent.dup
+            child.name = name
+            child.plugin = parent.plugin.dup
+            child.plugin.merge!(val)
+            @all_plugins << child
+            @all_names << child.name
+            next
+          end
+        end
+        @all_plugins.each do |p|
+          unless p.always_start || @all_names.include?(p.name)
+            Log.info("Plugin '#{p.name}' not started, because it " +
+                                "not 'always_start' and not in config file")
+            next
+          end
+          @plugins_to_run << p if p.runnable_by_config?
+        end
+        Log.debug("Plugins to start: #{@plugins_to_run.inspect}")
+        @plugins_to_run
+      end
+
+    end
+  end
+end

data/lib/kurchatov/plugin/dsl.rb

@@ -0,0 +1,81 @@
+module Kurchatov
+  module Plugins
+    class DSL
+
+      attr_reader :plugins
+      PLUGIN_EXT = '.rb'
+
+      def initialize
+        @plugins = Array.new
+      end
+
+      def last
+        @plugins.last
+      end
+
+      # plugins dsl part
+      def always_start(val)
+        last.always_start = !!val
+      end
+
+      def interval(val)
+        last.interval = val
+      end
+
+      def name(val)
+        last.name = val
+      end
+
+      def critical(val)
+        last.plugin[:critical] = val
+      end
+
+      def warning(val)
+        last.plugin[:warning] = val
+      end
+
+      def collect(opts = {}, *args, &block)
+        return unless last.respond_to_ohai?(opts)
+        last.collect = block
+      end
+
+      def run_if(opts = {}, &block)
+        return unless last.respond_to_ohai?(opts)
+        last.run_if = block
+      end
+
+      def last_plugin
+        last.plugin
+      end
+      alias :default :last_plugin
+
+
+      # load part
+
+      def self.load_riemann_plugins(paths)
+        dsl = Kurchatov::Plugins::DSL.new
+        paths.map do |path|
+          Log.error("Directory #{path} not exists") and exit Kurchatov::Config[:ERROR_CONFIG] unless File.directory? path 
+          Dir["#{path}/*#{PLUGIN_EXT}"].sort
+        end.flatten.each do |path|
+          begin
+            dsl.plugins << Kurchatov::Plugins::Riemann.new(File.basename(path, PLUGIN_EXT))
+            dsl.instance_eval(File.read(path), path)
+          rescue LoadError, SyntaxError => e
+            dsl.plugins.pop # todo: plugin.new creates
+            Log.error("Load plugin from file #{path}, #{e.class}: #{e}\n #{e.backtrace.join("\n")}")
+          end
+        end
+        dsl.plugins
+      end
+
+      def self.load_riemann_plugin(file)
+        dsl = Kurchatov::Plugins::DSL.new
+        dsl.plugins << Kurchatov::Plugins::Riemann.new(File.basename(file, PLUGIN_EXT))
+        dsl.instance_eval(File.read(file), file)
+        dsl.last
+      end
+
+    end
+  end
+end

data/lib/kurchatov/plugin/riemann.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+
+require "kurchatov/plugin"
+require "kurchatov/mashie"
+require "kurchatov/mixin/init"
+
+module Kurchatov
+  module Plugins
+    class Riemann < Kurchatov::Plugin
+
+      include Kurchatov::Mixin::Queue
+      include Kurchatov::Mixin::Ohai
+      include Kurchatov::Mixin::Event
+      include Kurchatov::Mixin::Command
+      include Kurchatov::Mixin::Http
+
+      attr_accessor :run_if, :collect, :always_start, :interval, :plugin
+
+      def initialize(name = '')
+        super(name)
+        @run_if = Proc.new {true}
+        @plugin = Mashie.new
+        @always_start = false
+        @collect = nil
+        @interval = 60
+      end
+
+      def run
+        loop do
+          t_start = Time.now
+          Timeout::timeout(interval * 2/3) do
+            self.instance_eval(&collect)
+          end
+          sleep(interval - (Time.now - t_start).to_i)
+        end
+      end
+
+      def respond_to_ohai?(opts = {})
+        opts.each { |k,v| return false unless ohai[k] == v }
+        true
+      end
+
+      def runnable_by_config?
+        Log.info("Plugin '#{self.name}' disabled by nil collect") and return if collect.nil?
+        Log.info("Plugin '#{self.name}' disabled in config") and return if plugin[:disable] == true
+        Log.info("Plugin '#{self.name}' not started by run_if condition ") and 
+          return if !self.instance_eval(&run_if)
+        @plugin[:service] = name if @plugin[:service].nil?
+        true
+      end
+
+    end
+  end
+end

data/lib/kurchatov/plugin.rb

@@ -0,0 +1,15 @@
+module Kurchatov
+  class Plugin
+
+    attr_accessor :name
+
+    def initialize(name)
+      @name = name
+    end
+
+    def run
+      raise
+    end
+
+  end
+end

data/lib/kurchatov/queue.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+module Kurchatov
+  class Queue
+    QUEUE_MAX_SIZE = 100
+
+    def initialize
+      @events = ::Queue.new
+    end
+
+    def <<(event)
+      if @events.size >= QUEUE_MAX_SIZE
+        drop = @events.shift
+        Log.error("Drop event: #{drop.inspect}. See Kurchatov::Queue::QUEUE_MAX_SIZE")
+      end
+      @events << event
+    end
+
+    def all
+      cur_events = Array.new
+      until @events.empty?
+        cur_events << @events.shift
+      end
+      cur_events
+    end
+
+  end
+end

data/lib/kurchatov/responders/http.rb

@@ -0,0 +1,36 @@
+module Kurchatov
+  module Responders
+    class Http < Kurchatov::Plugin
+
+      def initialize(conn)
+        @host, @port = conn.split(':')
+        @name = "http server #{@host}:#{@port}"
+        @s_time = Time.now
+      end
+
+      def run
+        @server ||= TCPServer.new(@host, @port)
+        loop do
+          client = @server.accept
+          response = info
+          headers = "HTTP/1.1 200 OK\r\n" +
+            "Server: Kurchatov Ruby\r\n" +
+            "Content-Length: #{response.bytesize}\r\n" +
+            "Content-Type: application/json\r\n\r\n"
+            client.print headers
+            client.print response
+            client.close
+        end
+      end
+
+      def info
+        {
+          :version => Kurchatov::VERSION,
+          :uptime => (Time.now - @s_time).to_i,
+          :config => Kurchatov::Config.to_hash,
+        }.to_json + "\n"
+      end
+
+    end
+  end
+end

data/lib/kurchatov/responders/init.rb

@@ -0,0 +1,3 @@
+require_relative "http"
+require_relative "riemann"
+require_relative "udp"