onering-agent 0.4.3

data/lib/onering/plugins/reporter.rb

@@ -0,0 +1,360 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'timeout'
+require 'optparse'
+require 'hashlib'
+require 'set'
+
+module Onering
+  class Reporter
+    DEFAULT_PLUGIN_GEMNAMES=[
+      'onering-report'
+    ]
+
+    DEFAULT_PLUGIN_PATH = [
+      '/var/lib/onering/reporter'
+    ]
+
+    DEFAULT_FACTER_PATH = [
+      '/etc/facter'
+    ]
+
+    DEFAULT_CACHE_FILE='/var/tmp/.onering-report-cache.json'
+    DEFAULT_CACHE_MAXAGE=600
+
+    include Onering::Util
+
+    attr_reader :facter_path
+
+    class PluginDelegate
+      def initialize(reporter, options={})
+        @_name = options.get(:plugin)
+        @_path = options.get(:path)
+
+        Onering::Logger.debug3("Creating plugin delegate for plugin #{@_name}", "Onering::Reporter::PluginDelegate")
+        @_reporter = reporter
+      end
+
+      def get_binding()
+        return binding()
+      end
+
+    # DSL methods
+    # -------------------------------------------------------------------------
+      def report(&block)
+        if block_given?
+          start = Time.now.to_f
+          yield
+
+          finish = (Time.now.to_f - start.to_f)
+          finish = (finish.round(4) rescue finish)
+          Onering::Logger.debug3("Finished evaluating report for plugin #{@_name}, took #{finish} seconds", "Onering::Reporter::PluginDelegate")
+        end
+      end
+
+      def property(name, value=nil)
+        @_reporter.property(name, value)
+      end
+
+      def stat(name, value=nil)
+        unless value.nil?
+          @_reporter.property((['metrics']+name.to_s.split('.')).join('.'), value)
+        end
+      end
+    end
+
+    def initialize(config={})
+      @options = config
+      @facter_path = DEFAULT_FACTER_PATH
+      @detected_gems = []
+
+      @path = [*Onering::Config.get('reporter.plugin_path',[])]
+      @path += DEFAULT_PLUGIN_PATH
+
+
+      begin
+        specs = Set.new()
+        @detected_gems = []
+
+        Gem::Specification.each do |spec|
+          specs << spec.name
+        end
+
+        @detected_gems = (specs.to_a.select{|i|
+          i =~ /^onering-report-/
+        } - DEFAULT_PLUGIN_GEMNAMES)
+      rescue Exception => e
+        Onering::Logger.warn("Unable to detect plugin gems: #{e.class.name} - #{e.message}", "Onering::Reporter")
+      end
+
+    # add gem paths to the @path
+      ([*Onering::Config.get('reporter.plugin_gems',[])]+@detected_gems+DEFAULT_PLUGIN_GEMNAMES).compact.each do |g|
+        begin
+          p = File.join(Util.gem_path(g), 'lib')
+          @path << File.join(p, 'reporter')
+          @facter_path << File.join(p, 'facter')
+        rescue Gem::LoadError => e
+          Onering::Logger.warn("Error loading gem: #{e.message}", "Onering::Reporter")
+          next
+        end
+      end
+
+      begin
+        ENV['FACTERLIB'] = @facter_path.join(':')
+        require 'facter'
+        Onering::Logger.debug("Facter loaded successfully, FACTERLIB is #{ENV['FACTERLIB']}", "Onering::Reporter")
+
+      rescue LoadError
+        Onering::Logger.error("Unable to load Facter library", "Onering::Reporter")
+      end
+    end
+
+    def load_plugins()
+
+    # load plugins from @path
+      @path.compact.uniq.each do |root|
+        begin
+          Dir["#{root}/*"].uniq.each do |directory|
+
+          # only process top-level directories
+            if File.directory?(directory)
+              d = File.basename(directory)
+
+              Onering::Logger.debug("Loading plugins from path #{directory}", "Onering::Reporter")
+
+            # allow plugins to be conditionally loaded based on fact values:
+            #   default - always load
+            #   <fact>-<fact_value> - load if <fact> == <fact_value>
+            #
+              if d == 'default' or Facter.value(d.split('-',2).first).to_s.downcase.nil_empty == d.split('-',2).last.to_s.downcase.nil_empty
+
+                Dir[File.join(directory, '*.rb')].each do |plugin|
+                  plugin = File.basename(plugin, '.rb')
+
+                  begin
+                    Timeout.timeout((@options[:plugin_timeout] || 10).to_i) do
+                      Onering::Logger.debug("Loading plugin #{directory}/#{plugin}.rb", "Onering::Reporter")
+                      Onering::Logger.debug3("Properties will be set in report object #{@_report.object_id}", "Onering::Reporter")
+                      eval(File.read("#{directory}/#{plugin}.rb"), PluginDelegate.new(self, {
+                        :plugin => plugin,
+                        :path   => "#{directory}/#{plugin}.rb"
+                      }).get_binding())
+                    end
+                  rescue Timeout::Error
+                    Onering::Logger.warn("Plugin #{plugin} took too long to return, skipping", "Onering::Reporter")
+                  end
+                end
+              end
+            end
+          end
+        rescue Exception => e
+          raise e if e.class === Timeout::Error
+
+          Onering::Logger.warn(e.message, "Onering::Reporter/#{e.class.name}")
+
+          e.backtrace.each do |eb|
+            Onering::Logger.debug(eb, "Onering::Reporter/#{e.class.name}")
+          end
+
+          next
+        end
+      end
+    end
+
+    def property(name, value=nil)      
+      unless value.nil?
+        Onering::Logger.debug3("-> Set property #{name.to_s} (was: #{@_report[:properties].get(name.to_s,'null')}) in object #{@_report.object_id}", "Onering::Reporter")
+        @_report[:properties].set(name.to_s, value)
+      end
+    end
+
+    def report(options={})
+      options = @options.merge(options)
+      @id = (@options[:id] || Onering::Config.get('id') || Onering::Config.get('reporter.fields.id') || Onering::Util.fact('hardwareid', nil))
+
+      if not @id.nil?
+        if options[:nocache]
+          return _generated_report()
+        else
+          rv = _cached_report(options)
+          return _generated_report() if rv.nil? or rv.empty?
+          return rv
+        end
+      else
+        Onering::Logger.fatal!("Cannot generate report without an ID", "Onering::Reporter")
+      end
+
+      return {}
+    end
+
+
+    def get(field, default=nil, options={})
+      if options[:data].is_a?(Hash)
+        _report = options[:data]
+      else
+        _report = self.report(options)
+      end
+
+    # this is kinda ugly
+    # because we don't know which property might have an @-prefix, progressively
+    # search through all of them.  first non-null match wins
+      parts       = field.to_s.split('.')
+
+    # create an array with every component of the path prefixed with the @-symbol, then with
+    # the path as is.
+    #
+    # e.g.:  onering report get metrics.disk.block
+    #        -> value exists in the inventory as properties.metrics.disk.@block,
+    #           but the user shouldn't need to know where that @-prefix is, so...
+    #
+    #        Search for all of these, first non-nil value wins:
+    #        * properties.metrics.disk.block
+    #        * properties.@metrics.disk.block
+    #        * properties.metrics.@disk.block
+    #        * properties.metrics.disk.@block
+    #        * metrics.disk.block
+    #
+      candidates  = [(['properties']+parts).join('.')]
+
+      parts.each_index{|ix|
+        candidates << (['properties']+(ix == 0 ? [] : parts[0..(ix-1)]) + ["@#{parts[ix]}"] + parts[ix+1..-1]).join('.')
+      }.flatten()
+
+      rv = nil
+
+    # search for the key using science or something
+      candidates.each do |c|
+        rv = _report.get(c)
+        break unless rv.nil?
+      end            
+
+    # if we're still nil by this point, use the fallback value
+      rv = _report.get(field) if rv.nil?
+
+    # attempt to get the value remotely if not found locally
+      if rv.nil? and not options[:local]
+        hid = Onering::Util.fact(:hardwareid)
+
+        if not hid.nil?
+          Onering::Logger.debug("Getting remote value #{field} for asset #{hid}")
+          return Onering::API.new(options[:api]).assets.get_field(hid, field, default)
+        end
+      end
+
+      return default if rv.nil?
+      return rv
+    end
+
+
+    def _generated_report()
+      Timeout.timeout((@options[:timeout] || 60).to_i) do
+        hostname = (Facter.value('fqdn') rescue %x{hostname -f}.strip.chomp)
+
+        @_report = {
+          :id           => @id,
+          :name         => hostname,
+          :aliases      => @options[:aliases],
+          :tags         => @options[:tags],
+          :status       => (@options[:status] || 'online'),
+          :inventory    => true,
+          :properties   => {}
+        }
+
+      # loads plugins and populates @_report
+        load_plugins()
+
+      # pull report field overrides from the config file
+        Onering::Config.get('reporter.fields',{}).each do |key, value|
+          Onering::Logger.debug("Override value #{key} from config file", "Onering::CLI::Report")
+
+          if value.is_a?(Hash)
+            value.coalesce(key, nil, '.').each do |k,v|
+              v = nil if ['null', '', '-'].include?(v.to_s.strip.chomp)
+              @_report = @_report.set(k, v)
+            end
+          else
+            value = nil if ['null', '', '-'].include?(value.to_s.strip.chomp)
+            @_report = @_report.set(key, value)
+          end
+        end
+
+      # return final report
+        return @_report.stringify_keys()
+      end
+
+      return {}
+    end
+
+    def _cached_report(options={})
+      options = @options.merge(options)
+      cachefile = (options[:cachefile] || DEFAULT_CACHE_FILE)
+      tries = 0
+
+      catch(:retry) do
+        tries += 1
+
+        if tries > 10
+          Onering::Logger.error("Too many retries reading cache #{cachefile}, generating report", "Onering::Reporter")
+          return _generated_report()
+        end
+
+        if File.readable?(cachefile)
+          Onering::Logger.debug("Loading cache file at #{cachefile}", "Onering::Reporter")
+          cache = File.read(cachefile)
+          cache = (MultiJson.load(cache) rescue {})
+
+          if _cache_expired?(cache, options[:maxage])
+            Onering::Logger.debug("Cache expired, regenerating", "Onering::Reporter")
+            throw :retry if _update_cache_file(cachefile)  
+          end
+
+          if options[:cacheregen] == true
+            Onering::Logger.debug("Forcing cache regeneration", "Onering::Reporter")
+            cache = _update_cache_file(cachefile)
+          end
+
+          if cache
+          # remove cached_at key
+            Onering::Logger.debug("Using cached data (#{Time.now.to_i - Time.parse(cache.get('cached_at')).to_i} seconds old)", "Onering::Reporter")
+            cache.delete('cached_at')
+            return cache
+          end
+        else
+          Onering::Logger.debug("Report cache file could not be read at #{cachefile}", "Onering::Reporter")
+          throw :retry if _update_cache_file(cachefile)
+        end
+      end
+
+      return {}
+    end
+
+
+    def _update_cache_file(cachefile=DEFAULT_CACHE_FILE)
+      begin
+        report = nil
+
+        File.open(cachefile, 'w+') do |file|
+          Onering::Logger.debug("Regenerating cache file at #{cachefile}", "Onering::Reporter")
+          report = _generated_report()
+          report['cached_at'] = Time.now.strftime('%Y-%m-%dT%H:%M:%S%z')
+          json = MultiJson.dump(report, :pretty => true)
+          file.puts(json)
+        end
+
+        return report
+      rescue Exception => e
+        Onering::Logger.info("Unable to write cache file #{cachefile}: #{e.class.name} - #{e.message}", "Onering::Reporter")
+        return false
+      end
+    end
+
+
+    def _cache_expired?(cache, age=DEFAULT_CACHE_MAXAGE)
+      if cache.is_a?(Hash)
+        return (Time.parse(cache.get('cached_at')) < (Time.now - age) rescue true)
+      else
+        return true
+      end
+    end
+  end
+end

data/lib/onering/util.rb

@@ -0,0 +1,150 @@
+module Onering
+  module Util
+    module String
+      def nil_empty
+        return nil if (self.strip.chomp.empty? rescue true)
+        self.strip.chomp
+      end
+
+      def to_bytes
+        case self
+        when /^\s*([\d\.]+)\s*([KMGTPEZY]?)([Bb])\s*$/
+          power = case $2
+          when 'K' then 1
+          when 'M' then 2
+          when 'G' then 3
+          when 'T' then 4
+          when 'P' then 5
+          when 'E' then 6
+          when 'Z' then 7
+          when 'Y' then 8
+          else 0
+          end
+
+          num = (Integer($1) rescue Float($1))
+          div = ($3 == 'b' ? 8 : 1)
+
+          return ((num * (1024 ** power)) / div)
+        else
+          return nil
+        end
+      end
+    end
+
+    extend self
+
+
+    HTTP_STATUS_CODES = {
+      400 => 'Bad Request',
+      401 => 'Unauthorized',
+      402 => 'Payment Required',
+      403 => 'Forbidden',
+      404 => 'Not Found',
+      405 => 'Method Not Allowed',
+      406 => 'Not Acceptable',
+      407 => 'Proxy Authentication Required',
+      408 => 'Request Timeout',
+      409 => 'Conflict',
+      410 => 'Gone',
+      411 => 'Length Required',
+      412 => 'Precondition Failed',
+      413 => 'Request Entity Too Large',
+      414 => 'Request-URI Too Long',
+      415 => 'Unsupported Media Type',
+      416 => 'Requested Range Not Satisfiable',
+      417 => 'Expectation Failed',
+      418 => 'I\'m a Teapot',
+      420 => 'Enhance Your Calm',
+      422 => 'Unprocessable Entity',
+      423 => 'Locked',
+      424 => 'Failed Dependency',
+      426 => 'Upgrade Required',
+      428 => 'Precondition Required',
+      429 => 'Too Many Requests',
+      431 => 'Request Header Fields Too Large',
+      444 => 'No Response',
+      451 => 'Unavailable For Legal Reasons',
+
+      500 => 'Internal Server Error',
+      501 => 'Not Implemented',
+      502 => 'Bad Gateway',
+      503 => 'Service Unavailable',
+      504 => 'Gateway Timeout',
+      505 => 'HTTP Version Not Supported',
+      508 => 'Loop Detected',
+      509 => 'Bandwidth Limit Exceeded',
+      510 => 'Not Extended',
+      511 => 'Network Authentication Required'
+    }
+
+    def gem_path(name)
+      if Gem::Specification.respond_to?(:find_by_name)
+        return Gem::Specification.find_by_name(name).gem_dir
+      else
+        return Gem::SourceIndex.from_installed_gems.find_name(name).sort{|a,b|
+          a.version.to_s <=> b.version.to_s
+        }.last.full_gem_path
+      end
+    end
+
+    def fact(name, default=nil)
+      reporter = Onering::Reporter.new()
+      name = name.to_s
+
+      if defined?(Facter)
+        if name.downcase == 'all'
+          return Facter.to_hash()
+        else
+          fact_value = Facter.value(name)
+
+        # short circuit nil responses
+          return default if fact_value.nil?
+
+        # if we are asking for a nested object...
+          if name.include?('.')
+          # ...and the response IS an object...
+            if fact_value.is_a?(Hash)
+            # remove the first part and return the rest
+              name = name.sub(/^[^\.]+\./,'')
+              return fact_value.get(name, default)
+            else
+            # not an object, return default
+              return default
+            end
+          else
+        # this is a simple request, return the fact
+            return fact_value
+          end
+        end
+      end
+
+      return default
+    end
+
+    def make_filter(filter)
+      filter = filter.collect{|k,v| "#{k}/#{v}" } if filter.is_a?(Hash)
+      filter = filter.collect{|i| i.sub(':','/') }.join("/") if filter.is_a?(Array)
+      return filter
+    end
+
+    def http_status(code)
+      return (HTTP_STATUS_CODES[code.to_i] || nil)
+    end
+  end
+end
+
+class String
+  include Onering::Util::String
+end
+
+class Module
+  def submodules
+    constants.collect {|const_name| const_get(const_name)}.select {|const| const.class == Module}
+  end
+end
+
+class NilClass
+  def <=>(*args)
+    return 1
+  end
+end