riemann-babbler 1.4.0 → 2.0.0pre1

data/.gitignore

@@ -2,3 +2,5 @@ pkg/
 .idea
 .rvmrc
 *.swp
+.bundle
+vendor

data/Gemfile

@@ -1,18 +1,4 @@
 source 'https://rubygems.org'
 
-gem 'riemann-client'
-gem 'configatron'
-gem 'logger'
-gem 'trollop'
-gem 'sys-filesystem'
-gem 'rest-client'
-gem 'net-ping'
-gem 'file-tail'
-gem 'sys-proctable'
-gem 'net-http-server'
-
-gem 'rspec'
-gem 'rake'
-
-gem 'pry'
-gem 'awesome_print'
+# Specify your gem's dependencies in riemann-babbler.gemspec
+gemspec

data/Gemfile.lock

@@ -1,50 +1,46 @@
+PATH
+  remote: .
+  specs:
+    riemann-babbler (2.0.0pre1)
+      configatron
+      docile
+      net-http-server
+      rest-client
+      riemann-client
+      sys-filesystem
+      trollop
+
 GEM
   remote: https://rubygems.org/
   specs:
-    awesome_print (1.1.0)
-    beefcake (0.3.7)
+    beefcake (0.4.0)
     blankslate (2.1.2.4)
     coderay (1.0.9)
-    configatron (2.10.0)
+    configatron (2.13.0)
       yamler (>= 0.1.0)
-    diff-lcs (1.2.2)
-    ffi (1.4.0)
-    file-tail (1.0.12)
-      tins (~> 0.5)
-    logger (1.2.8)
-    method_source (0.8.1)
-    mime-types (1.21)
+    docile (1.1.0)
+    ffi (1.9.0)
+    method_source (0.8.2)
+    mime-types (1.25)
     mtrc (0.0.4)
     net-http-server (0.2.2)
       parslet (~> 1.0)
-    net-ping (1.6.0)
-      ffi (>= 1.0.0)
     parslet (1.5.0)
       blankslate (~> 2.0)
-    pry (0.9.12)
+    pry (0.9.12.2)
       coderay (~> 1.0.5)
       method_source (~> 0.8)
       slop (~> 3.4)
-    rake (10.0.4)
+    rake (10.1.0)
     rest-client (1.6.7)
       mime-types (>= 1.16)
-    riemann-client (0.2.1)
+    riemann-client (0.2.2)
       beefcake (>= 0.3.5)
       mtrc (>= 0.0.4)
       trollop (>= 1.16.2)
-    rspec (2.13.0)
-      rspec-core (~> 2.13.0)
-      rspec-expectations (~> 2.13.0)
-      rspec-mocks (~> 2.13.0)
-    rspec-core (2.13.1)
-    rspec-expectations (2.13.0)
-      diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.13.0)
-    slop (3.4.3)
-    sys-filesystem (1.1.0)
+    slop (3.4.6)
+    sys-filesystem (1.1.1)
       ffi
-    sys-proctable (0.9.3)
-    tins (0.8.0)
     trollop (2.0)
     yamler (0.1.0)
 
@@ -52,17 +48,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  awesome_print
-  configatron
-  file-tail
-  logger
-  net-http-server
-  net-ping
+  bundler (~> 1.3)
   pry
   rake
-  rest-client
-  riemann-client
-  rspec
-  sys-filesystem
-  sys-proctable
-  trollop
+  riemann-babbler!

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Vasiliev Dmitry
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -8,14 +8,24 @@ gem install riemann-babbler
 ### Use
 ```
 $ riemann-babbler --help
-Riemann-babbler is plugin manager for riemann-tools.
+Riemann-babbler is tool for monitoring with riemann.
 
 Usage:
        riemann-babbler [options]
 where [options] are:
-  --config, -c:   Config file (default: /etc/riemann-babbler/config.yml)
-  --version, -v:   Print version and exit
-  --help, -h:   Show this message
+             --config, -c <s>:   Config file (default: /etc/riemann-babbler/config.yml)
+               --host, -h <s>:   Riemann host (default: 127.0.0.1)
+               --port, -p <i>:   Riemann port (default: 5555)
+            --timeout, -t <i>:   Riemann timeout (default: 5)
+        --fqdn, --no-fqdn, -f:   Use fqdn for event hostname (default: true)
+                --ttl, -l <i>:   TTL for events (default: 60)
+           --interval, -i <i>:   Seconds between updates (default: 60)
+          --log-level, -o <s>:   Level log (default: DEBUG)
+  --plugins-directory, -u <s>:   Directory for plugins (default: /usr/share/riemann-babbler/plugins)
+              --tcp, --no-tcp:   Use TCP transport instead of UDP (improves reliability, slight overhead. (Default: true)
+     --responder-port, -r <i>:   Port to bind responder (default: 55755)
+                --version, -v:   Print version and exit
+                   --help, -e:   Show this message
 ```
 
 ### Config
@@ -27,14 +37,6 @@ riemann:
   tags: 
     - prod
     - web
-  suffix: ".testing"
-  preffix: "prefix"
-
-plugins:
-  dirs:
-    - /etc/riemann/plugins # load all rb files in dirs
-  files:
-    - /var/lib/riemann-plugins/test.rb # and custom load somefile
 ```
 ##### Config yml for custom plugin
 ```yaml
@@ -51,7 +53,7 @@ plugins:
 ### Custom Plugin
 #### Example 1
 ```ruby
-class Riemann::Babbler::Awesomeplugin < Riemann::Babbler
+class Riemann::Babbler::Plugin::AwesomePlugin < Riemann::Babbler::Plugin
 
   def init
     plugin.set_default(:service, 'awesome plugin' )
@@ -70,7 +72,7 @@ end
 ```
 #### Example 2
 ```ruby
-class Riemann::Babbler::Awesomeplugin < Riemann::Babbler
+class Riemann::Babbler::Plugin::AwesomePlugin < Riemann::Babbler::Plugin
 
   def init
     plugin.set_default(:service, 'awesome plugin' )
@@ -94,9 +96,9 @@ class Riemann::Babbler::Awesomeplugin < Riemann::Babbler
     status <<  {
         :service => plugin.service + " cmd2",
         :metric => shell plugin.cmd2, # shell - helper
-        :as_diff => true # report as diffencial: current - last
+        :as_diff => true # report as differential: current - last
       }
     status
   end
 end
-```
+```

data/bin/riemann-babbler

@@ -2,24 +2,44 @@
 # coding: utf-8
 
 require 'trollop'
-require 'configatron'
+require 'yaml'
+require_relative '../lib/riemann/babbler'
+require_relative '../lib/riemann/babbler/plugin'
 
-$:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-require 'riemann/babbler/version'
-require 'riemann/babbler/start'
+include Riemann::Babbler::Logging
+include Riemann::Babbler::Options
 
-opts = Trollop::options do
+cmd_opts = Trollop::options do
   version "Riemann babbler #{Riemann::Babbler::VERSION}"
   banner <<-EOS
-Riemann-babbler is plugin manager for riemann-tools.
+Riemann-babbler is tool for monitoring with riemann.
 
 Usage:
-       riemann-babbler [options] 
+       riemann-babbler [options]
 where [options] are:
-EOS
+  EOS
 
   opt :config, 'Config file', :default => '/etc/riemann-babbler/config.yml'
+  opt :host, 'Riemann host', :default => '127.0.0.1'
+  opt :port, 'Riemann port', :default => 5555
+  opt :timeout, 'Riemann timeout', :default => 5
+  #todo: opt :event_hostname, 'Event hostname', :type => String
+  opt :fqdn, 'Use fqdn for event hostname', :default => true
+  opt :ttl, 'TTL for events', :default => 60
+  opt :interval, 'Seconds between updates', :default => 60
+  opt :log_level, 'Level log', :default => 'DEBUG'
+  #todo: opt :log_output, 'Directions to puts your log', :default => STDOUT
+  #todo: opt :log_format, 'Log format', :default => '%Y-%m-%d %H:%M:%S'
+  opt :plugins_directory, 'Directory for plugins', :default => '/usr/share/riemann-babbler/plugins'
+  opt :tcp, 'Use TCP transport instead of UDP (improves reliability, slight overhead.', :default => true
+  opt :responder_port, 'Port to bind responder', :default => 55755
 end
 
-babbler = Riemann::Babbler::Starter.new(opts, configatron)
-babbler.start!
+# load current settings
+opts.configure_from_hash({ :riemann => cmd_opts })
+@@logger.level = Logger.const_get(opts.riemann.log_level) #todo: to hard
+merge_config(opts.riemann.config)
+
+# start
+Riemann::Babbler::Responder.new.run!
+Riemann::Babbler::PluginLoader.new(Riemann::Babbler::Sender.new).run!

data/lib/riemann/babbler/errors.rb

@@ -0,0 +1,13 @@
+module Riemann
+  module Babbler
+
+    module Errors
+      RESOLV_RIEMANN_SERVER = 1
+      INIT_CONNECT          = 2
+      CONNECTION_PROBLEM    = 3
+    end
+
+  end
+end
+
+

data/lib/riemann/babbler/logging.rb

@@ -0,0 +1,36 @@
+require 'logger'
+
+module Riemann
+  module Babbler
+
+    module Logging
+
+      @levels  = { 'DEBUG' => 0, 'INFO' => 1, 'WARN' => 2, 'ERROR' => 3, 'FATAL' => 4 }
+      @@logger = Logger.new(STDOUT) #todo: opts
+
+      def log(log_level, message, method = nil)
+        speaker = get_logger_speaker
+        speaker = speaker + "##{method}" unless method.nil?
+        @@logger.send(log_level.to_sym, " [#{speaker}] #{message}")
+      end
+
+      def set_logger_speaker(speaker)
+        @logger_speaker = speaker
+      end
+
+      def get_logger_speaker
+        if @logger_speaker.nil?
+          self.class == Class ? "C: #{self.to_s}" : "I: #{self.class.to_s}"
+        else
+          @logger_speaker
+        end
+      end
+
+      def self.included(base)
+        base.extend(self)
+      end
+
+    end
+
+  end
+end

data/lib/riemann/babbler/{support/deep_merge.rb → monkey_patches.rb}

@@ -1,3 +1,28 @@
+module Beefcake
+  class Buffer
+
+    def initialize(buf='')
+      if buf.respond_to?(:force_encoding)
+        self.buf = buf.force_encoding('BINARY')
+      else
+        self.buf = buf
+      end
+    end
+
+    def append_string(s)
+      append_uint64(s.length)
+      if s.respond_to?(:force_encoding)
+        self << s.force_encoding('BINARY')
+      else
+        self << s
+      end
+    end
+
+    alias :append_bytes :append_string
+
+  end
+end
+
 # https://raw.github.com/Offirmo/hash-deep-merge/master/lib/hash_deep_merge.rb
 class Hash
 
@@ -13,36 +38,20 @@ class Hash
 
   protected
 
-  # better, recursive, preserving method
-  # OK OK this is not the most efficient algorithm,
-  # but at last it's *perfectly clear and understandable*
-  # so fork and improve if you need 5% more speed, ok ?
   def internal_deep_merge!(source_hash, specialized_hash)
-
-    #puts "starting deep merge..."
-
     specialized_hash.each_pair do |rkey, rval|
-      #puts "   potential replacing entry : " + rkey.inspect
-
       if source_hash.has_key?(rkey) then
-        #puts "   found potentially conflicting entry for #{rkey.inspect} : #{rval.inspect}, will merge :"
         if rval.is_a?(Hash) and source_hash[rkey].is_a?(Hash) then
-          #puts "      recursing..."
           internal_deep_merge!(source_hash[rkey], rval)
         elsif rval == source_hash[rkey] then
-          #puts "      same value, skipping."
         else
-          #puts "      replacing."
           source_hash[rkey] = rval
         end
       else
-        #puts "   found new entry #{rkey.inspect}, adding it..."
         source_hash[rkey] = rval
       end
     end
 
-    #puts "deep merge done."
-
     source_hash
   end
 end

data/lib/riemann/babbler/options.rb

@@ -0,0 +1,43 @@
+require 'configatron'
+
+module Riemann
+  module Babbler
+
+    module Options
+
+      @@configatron = configatron
+
+      def opts
+        @@configatron
+      end
+
+      #@return
+      def merge_config(file)
+        config_from_file = if File.exist?(file)
+                             YAML.load_file(file).to_hash
+                           else
+                             log :error, "Can't load config file #{file}"
+                             Hash.new
+                           end
+        config_default   = opts.to_hash
+        result_config    = config_default.deep_merge(config_from_file)
+        opts.configure_from_hash(result_config)
+      end
+
+      def name_to_underscore(name = 'Riemann::Babbler::Plugin::TwCli')
+        name.split('::').last.gsub(/(\p{Lower})(\p{Upper})/, "\\1_\\2").downcase
+      end
+
+      def underscore_to_name(underscore = 'tw_cli')
+        underscore.split('_').map { |part| part.capitalize }.join('')
+      end
+
+      def self.included(base)
+        Configatron.log.level = Logger::FATAL
+        base.extend(self)
+      end
+
+    end
+
+  end
+end

data/lib/riemann/babbler/plugin.rb

@@ -0,0 +1,154 @@
+#encoding: utf-8
+
+require 'riemann/babbler/plugins/helpers/init'
+
+module Riemann
+  module Babbler
+
+    module State
+      OK       = 'ok'
+      WARNING  = 'warning'
+      CRITICAL = 'critical'
+    end
+
+    class Plugin
+
+      include Riemann::Babbler::Logging
+      include Riemann::Babbler::Errors
+      include Riemann::Babbler::Options
+      include Riemann::Babbler::Plugins::Helpers
+
+      def self.registered_plugins
+        @plugins ||= []
+      end
+
+      def self.inherited(klass)
+        registered_plugins << klass
+      end
+
+      attr_reader :riemann, :plugin_name, :plugin, :errors
+
+      def initialize(riemann)
+        @riemann     = riemann
+        @storage     = Hash.new
+        @plugin_name = name_to_underscore(self.class.name)
+        @plugin      = opts.plugins.send(plugin_name)
+        @errors      = opts.errors.send(plugin_name)
+        set_default
+        init
+      end
+
+      def set_default
+        plugin.set_default(:interval, opts.riemann.interval)
+        plugin.timeout        = (plugin.interval * 2).to_f/3
+        errors.last_error     = nil
+        errors.last_error_at  = nil
+        errors.has_last_error = false
+        errors.reported       = true
+        plugin_no_error!
+        errors.count = 0
+      end
+
+      def init
+        # nothing to do
+      end
+
+      def run_plugin
+        if plugin.run.nil?
+          true
+        else
+          plugin.run ? true : false
+        end
+      end
+
+      def tick
+        posted_array = collect
+        posted_array = posted_array.class == Array ? posted_array : [posted_array]
+        posted_array.uniq.each { |event| report event }
+      end
+
+      def report(event)
+        report_with_diff(event) and return if event[:as_diff]
+        event[:metric] = event[:metric].round(2) if event[:metric].kind_of? Float
+        event[:state] = get_state(event)
+        riemann << event
+      end
+
+      def report_with_diff(event)
+        current_metric = event[:metric]
+        old_metric     = @storage[event[:service]]
+        if old_metric && current_metric + old_metric < 2**64
+          event[:metric] = current_metric - old_metric
+          event.delete(:as_diff)
+          report(event)
+        end
+        @storage[event[:service]] = current_metric
+      end
+
+      #@return state
+      def get_state(event)
+        return event[:state] if event[:state]
+        metric = event[:metric].to_f
+        return State::CRITICAL if metric.nil? #todo: неправильно здесь возврашать nil, это ошибка плагина!!!
+        warning  = plugin.states.warning.nil? ? nil : plugin.states.warning
+        critical = plugin.states.critical.nil? ? nil : plugin.states.critical
+        return State::OK if (warning || critical).nil?
+        if warning && critical
+          return case
+                   when metric.between?(warning, critical)
+                     State::WARNING
+                   when metric > warning
+                     State::CRITICAL
+                   else
+                     State::OK
+                 end
+        end
+        if warning
+          return (metric >= warning) ? State::WARNING : State::OK
+        end
+        if critical
+          return (metric >= critical) ? State::CRITICAL : State::OK
+        end
+      end
+
+      def plugin_error!(msg)
+        errors.count += 1
+        log :error, "#{plugin.service} error num #{errors.count}:\n #{msg}"
+        errors.has_last_error = true
+        errors.reported       = false
+        errors.last_error_at  = Time.now
+        errors.last_error     = "#{msg}"
+      end
+
+      def plugin_no_error!
+        if errors.has_last_error
+          log :error, "#{plugin.service} error num #{errors.count} fixed"
+          errors.has_last_error = false
+        end
+      end
+
+      # Main method
+      def run!
+        return 0 unless run_plugin
+        t0 = Time.now
+        loop do
+
+          begin
+            Timeout::timeout(plugin.interval) { tick }
+          rescue TimeoutError
+            plugin_error!('Timeout plugin execution')
+          rescue => e
+            plugin_error!("Plugin '#{plugin_name}' has a error.\n #{e.class}: #{e}\n #{e.backtrace.join("\n")}")
+          else
+            plugin_no_error!
+          end
+
+          sleep(plugin.interval - ((Time.now - t0) % plugin.interval))
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/riemann/babbler/plugin_loader.rb

@@ -0,0 +1,86 @@
+module Riemann
+  module Babbler
+    class PluginLoader
+
+      include Riemann::Babbler::Logging
+      include Riemann::Babbler::Options
+
+      AUTO_START = [
+          'cpu',
+          'cpu_fan',
+          'cpu_temp',
+          'disk',
+          'disk_stat',
+          'exim4',
+          'la',
+          'mdadm',
+          'mega_cli',
+          'memory',
+          'net',
+          'runit',
+          'tw_cli',
+          'errors_reporter'
+      ].freeze
+
+      attr_accessor :sender
+
+      def initialize(riemann)
+        @sender = riemann
+      end
+
+      def all_available_plugins
+        default_plugins_dir = File.expand_path('../plugins/', __FILE__)
+        plugins             = []
+        Dir.glob(default_plugins_dir + '/*.rb') do |file|
+          plugins << file
+        end
+        if Dir.exist? opts.riemann.plugins_directory
+          Dir.glob(opts.riemann.plugins_directory + '/*.rb') do |file|
+            plugins << file
+          end
+        else
+          log :error, "Directory doesn't exists: #{opts.riemann.plugins_directory}"
+        end
+        plugins
+      end
+
+      def require_all_plugins
+        Riemann::Babbler::Plugin.registered_plugins.clear
+        all_available_plugins.each { |file| require file }
+        log :debug, "Require plugins: #{Riemann::Babbler::Plugin.registered_plugins}"
+        # load parent
+        opts.plugins.to_hash.each do |plugin_name, plugin_opts|
+          next if plugin_opts.nil?
+          next unless plugin_opts.kind_of?(Hash)
+          if plugin_opts.has_key? "parent"
+            klass = Class.new(underscore_to_name(plugin_name))
+            klass.send(:title, underscore_to_name(plugin_opts[:parent]).to_sym)
+            Riemann::Babbler::Plugin.registered_plugins << klass
+          end
+        end
+      end
+
+      def run!
+        plugin_names_to_run = AUTO_START + opts.plugins.to_hash.keys
+        require_all_plugins
+        started_plugins = []
+        Riemann::Babbler::Plugin.registered_plugins.each do |klass|
+          if plugin_names_to_run.include? name_to_underscore(klass.to_s)
+            started_plugins << klass
+          end
+        end
+        plugin_threads = started_plugins.map do |plugin|
+          Thread.new {
+            log :unknown, "Start plugin #{plugin}"
+            plugin.new(sender).run!
+          }
+        end
+        Signal.trap 'TERM' do
+          plugin_threads.each(&:kill)
+        end
+        plugin_threads.each(&:join)
+      end
+
+    end
+  end
+end