mcollective-client 2.0.0 → 2.2.0

data/lib/mcollective.rb

@@ -11,6 +11,7 @@ require 'rbconfig'
 require 'tempfile'
 require 'tmpdir'
 require 'mcollective/monkey_patches'
+require 'mcollective/cache'
 
 # == The Marionette Collective
 #
@@ -23,42 +24,50 @@ require 'mcollective/monkey_patches'
 #   http://www.devco.net/archives/2009/10/18/middleware_for_systems_administration.php
 module MCollective
   # Exceptions for the RPC system
+  class DDLValidationError<RuntimeError;end
+  class ValidatorError<RuntimeError; end
+  class MsgDoesNotMatchRequestID < RuntimeError; end
+  class MsgTTLExpired<RuntimeError;end
+  class NotTargettedAtUs<RuntimeError;end
   class RPCError<StandardError;end
+  class SecurityValidationFailed<RuntimeError;end
+
+  class InvalidRPCData<RPCError;end
+  class MissingRPCData<RPCError;end
   class RPCAborted<RPCError;end
   class UnknownRPCAction<RPCError;end
-  class MissingRPCData<RPCError;end
-  class InvalidRPCData<RPCError;end
   class UnknownRPCError<RPCError;end
-  class NotTargettedAtUs<RuntimeError;end
-  class SecurityValidationFailed<RuntimeError;end
-  class DDLValidationError<RuntimeError;end
-  class MsgTTLExpired<RuntimeError;end
-  class MsgDoesNotMatchRequestID < RuntimeError; end
 
-
-  autoload :Config, "mcollective/config"
-  autoload :Log, "mcollective/log"
-  autoload :Logger, "mcollective/logger"
-  autoload :Runner, "mcollective/runner"
-  autoload :RunnerStats, "mcollective/runnerstats"
+  autoload :Agent, "mcollective/agent"
   autoload :Agents, "mcollective/agents"
+  autoload :Aggregate, "mcollective/aggregate"
+  autoload :Application, "mcollective/application"
+  autoload :Applications, "mcollective/applications"
   autoload :Client, "mcollective/client"
-  autoload :Util, "mcollective/util"
-  autoload :Optionparser, "mcollective/optionparser"
+  autoload :Config, "mcollective/config"
   autoload :Connector, "mcollective/connector"
-  autoload :Security, "mcollective/security"
+  autoload :Data, "mcollective/data"
+  autoload :DDL, "mcollective/ddl"
+  autoload :Discovery, "mcollective/discovery"
   autoload :Facts, "mcollective/facts"
-  autoload :Registration, "mcollective/registration"
-  autoload :PluginManager, "mcollective/pluginmanager"
-  autoload :RPC, "mcollective/rpc"
+  autoload :Logger, "mcollective/logger"
+  autoload :Log, "mcollective/log"
   autoload :Matcher, "mcollective/matcher"
   autoload :Message, "mcollective/message"
+  autoload :Optionparser, "mcollective/optionparser"
+  autoload :Generators, "mcollective/generators"
+  autoload :PluginManager, "mcollective/pluginmanager"
+  autoload :PluginPackager, "mcollective/pluginpackager"
+  autoload :Registration, "mcollective/registration"
+  autoload :RPC, "mcollective/rpc"
+  autoload :Runner, "mcollective/runner"
+  autoload :RunnerStats, "mcollective/runnerstats"
+  autoload :Security, "mcollective/security"
+  autoload :Shell, "mcollective/shell"
   autoload :SSL, "mcollective/ssl"
-  autoload :Application, "mcollective/application"
-  autoload :Applications, "mcollective/applications"
+  autoload :Util, "mcollective/util"
+  autoload :Validator, "mcollective/validator"
   autoload :Vendor, "mcollective/vendor"
-  autoload :Shell, "mcollective/shell"
-  autoload :PluginPackager, "mcollective/pluginpackager"
 
   MCollective::Vendor.load_vendored
 

data/lib/mcollective/agent.rb

@@ -0,0 +1,5 @@
+module MCollective
+  module Agent
+
+  end
+end

data/lib/mcollective/agents.rb

@@ -54,6 +54,11 @@ module MCollective
         if activate_agent?(agentname)
           PluginManager << {:type => "#{agentname}_agent", :class => classname, :single_instance => single_instance}
 
+          # Attempt to instantiate the agent once so any validation and hooks get run
+          # this does a basic sanity check on the agent as a whole, if this fails it
+          # will be removed from the plugin list
+          PluginManager["#{agentname}_agent"]
+
           Util.subscribe(Util.make_subscriptions(agentname, :broadcast)) unless @@agents.include?(agentname)
 
           @@agents[agentname] = {:file => agentfile}
@@ -108,22 +113,6 @@ module MCollective
       PluginManager.include?("#{agentname}_agent")
     end
 
-    # Returns the help for an agent after first trying to get
-    # rid of some indentation infront
-    def help(agentname)
-      raise("No such agent") unless include?(agentname)
-
-      body = PluginManager["#{agentname}_agent"].help.split("\n")
-
-      if body.first =~ /^(\s+)\S/
-        indent = $1
-
-        body = body.map {|b| b.gsub(/^#{indent}/, "")}
-      end
-
-      body.join("\n")
-    end
-
     # Dispatches a message to an agent, accepts a block that will get run if there are
     # any replies to process from the agent
     def dispatch(request, connection)

data/lib/mcollective/aggregate.rb

@@ -0,0 +1,61 @@
+module MCollective
+  class Aggregate
+    autoload :Result, 'mcollective/aggregate/result'
+    autoload :Base, 'mcollective/aggregate/base'
+
+    attr_accessor :ddl, :functions, :action
+
+    def initialize(ddl)
+      @functions = []
+      @ddl = ddl
+      @action = ddl[:action]
+
+      create_functions
+    end
+
+    # Creates instances of the Aggregate functions and stores them in the function array.
+    # All aggregate call and summarize method calls operate on these function as a batch.
+    def create_functions
+      @ddl[:aggregate].each_with_index do |agg, i|
+        contains_output?(agg[:args][0])
+
+        output = agg[:args][0]
+        arguments = agg[:args][1..(agg[:args].size)]
+
+        @functions << load_function(agg[:function]).new(output, arguments, agg[:format], @action)
+      end
+    end
+
+    # Check if the function param is defined as an output for the action in the ddl
+    def contains_output?(output)
+      raise "'#{@ddl[:action]}' action does not contain output '#{output}'" unless @ddl[:output].keys.include?(output)
+    end
+
+    # Call all the appropriate functions with the reply data received from RPC::Client
+    def call_functions(reply)
+      @functions.each do |function|
+        Log.debug("Calling aggregate function #{function} for result")
+        function.process_result(reply[:data][function.output_name], reply)
+      end
+    end
+
+    # Finalizes the function returning a result object
+    def summarize
+      summary = @functions.map do |function|
+        function.summarize
+      end
+
+      summary.sort{|x,y| x.result[:output] <=> y.result[:output]}
+    end
+
+    # Loads function from disk for use
+    def load_function(function_name)
+      function_name = function_name.to_s.capitalize
+
+      PluginManager.loadclass("MCollective::Aggregate::#{function_name}") unless Aggregate.const_defined?(function_name)
+      Aggregate.const_get(function_name)
+    rescue Exception
+      raise "Aggregate function file '#{function_name.downcase}.rb' cannot be loaded"
+    end
+  end
+end

data/lib/mcollective/aggregate/base.rb

@@ -0,0 +1,40 @@
+module MCollective
+  class Aggregate
+    class Base
+      attr_accessor :name, :result, :output_name, :action, :aggregate_format, :arguments
+
+      def initialize(output_name, arguments, aggregate_format, action)
+        @name = self.class.to_s
+        @output_name = output_name
+
+        # Any additional arguments passed in the ddl after the output field will
+        # be stored in the arguments array which can be used in the function
+        @arguments = arguments
+        @aggregate_format = aggregate_format
+        @action = action
+        @result = {:value => nil, :type => nil, :output => output_name}
+
+        startup_hook
+      end
+
+      ['startup_hook', 'process_result'].each do |method|
+        define_method method do
+          raise RuntimeError, "'#{method}' method of function class #{@name} has not yet been implemented"
+        end
+      end
+
+      # Stops execution of the function and returns a specific ResultObject,
+      # aggregate functions will most likely override this but this is the simplest
+      # case so we might as well default to that
+      def summarize
+        raise "Result type is not set while trying to summarize aggregate function results" unless @result[:type]
+
+        result_class(@result[:type]).new(@result, @aggregate_format, @action)
+      end
+
+      def result_class(type)
+        Result.const_get("#{type.to_s.capitalize}Result")
+      end
+    end
+  end
+end

data/lib/mcollective/aggregate/result.rb

@@ -0,0 +1,9 @@
+module MCollective
+  class Aggregate
+    module Result
+      autoload :Base, 'mcollective/aggregate/result/base'
+      autoload :NumericResult, 'mcollective/aggregate/result/numeric_result'
+      autoload :CollectionResult, 'mcollective/aggregate/result/collection_result'
+    end
+  end
+end

data/lib/mcollective/aggregate/result/base.rb

@@ -0,0 +1,25 @@
+module MCollective
+  class Aggregate
+    module Result
+      class Base
+        attr_accessor :result, :aggregate_format, :action
+
+        def initialize(result, aggregate_format, action)
+          raise "No aggregate_format defined in ddl or aggregate function" unless aggregate_format
+
+          @result = result
+          @aggregate_format = aggregate_format
+          @action = action
+        end
+
+        def to_s
+          raise "'to_s' method not implemented for result class '#{self.class}'"
+        end
+
+        def result_type
+          @result[:type]
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/aggregate/result/collection_result.rb

@@ -0,0 +1,19 @@
+module MCollective
+  class Aggregate
+    module Result
+      class CollectionResult<Base
+        def to_s
+          return "" if @result[:value].keys.include?(nil)
+
+          result = StringIO.new
+
+          @result[:value].sort{|x,y| x[1] <=> y[1]}.reverse.each do |value|
+            result.puts "    " + @aggregate_format % [value[0], value[1]]
+          end
+
+          result.string.chomp
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/aggregate/result/numeric_result.rb

@@ -0,0 +1,13 @@
+module MCollective
+  class Aggregate
+    module Result
+      class NumericResult<Base
+        def to_s
+          return "" if @result[:value].nil?
+
+          return @aggregate_format % @result[:value]
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/application.rb

@@ -171,12 +171,12 @@ module MCollective
           end
 
           # type was given and its not one of our special types, just pass it onto optparse
-          opts_array << carg[:type] if carg[:type] and ! [:bool, :array].include?(carg[:type])
+          opts_array << carg[:type] if carg[:type] && ![:boolean, :bool, :array].include?(carg[:type])
 
           opts_array << carg[:description]
 
           # Handle our special types else just rely on the optparser to handle the types
-          if carg[:type] == :bool
+          if [:bool, :boolean].include?(carg[:type])
             parser.send(*opts_array) do |v|
               validate_option(carg[:validate], carg[:name], v)
 
@@ -255,9 +255,11 @@ module MCollective
         raise(e)
       end
 
-      err_dest.puts "#{$0} failed to run: #{e} (#{e.class})"
+      err_dest.puts "\nThe %s application failed to run, use -v for full error details: %s\n" % [ Util.colorize(:bold, $0), Util.colorize(:red, e.to_s)]
 
       if options.nil? || options[:verbose]
+        e.backtrace.first << Util.colorize(:red, "  <----")
+        err_dest.puts "\n%s %s" % [ Util.colorize(:red, e.to_s), Util.colorize(:bold, "(#{e.class.to_s})")]
         e.backtrace.each{|l| err_dest.puts "\tfrom #{l}"}
       end
 
@@ -307,7 +309,7 @@ module MCollective
     # Exit with 0 if no discovery were done and > 0 responses were received
     # Exit with 1 if no nodes were discovered
     # Exit with 2 if nodes were discovered but some RPC requests failed
-    # Exit with 3 if nodes were discovered, but not responses receivedif
+    # Exit with 3 if nodes were discovered, but not responses received
     # Exit with 4 if no discovery were done and no responses were received
     def halt(stats)
       request_stats = {:discoverytime => 0,
@@ -347,6 +349,7 @@ module MCollective
     # cli options wouldnt take effect which could have a disasterous outcome
     def rpcclient(agent, flags = {})
       flags[:options] = options unless flags.include?(:options)
+      flags[:exit_on_failure] = false
 
       super
     end

data/lib/mcollective/applications.rb

@@ -11,10 +11,10 @@ module MCollective
       begin
         load_application(appname)
       rescue Exception => e
-        e.backtrace.first << RPC::Helpers.colorize(:red, "  <----")
+        e.backtrace.first << Util.colorize(:red, "  <----")
         STDERR.puts "Application '#{appname}' failed to load:"
         STDERR.puts
-        STDERR.puts RPC::Helpers.colorize(:red, "   #{e} (#{e.class})")
+        STDERR.puts Util.colorize(:red, "   #{e} (#{e.class})")
         STDERR.puts
         STDERR.puts "       %s" % [e.backtrace.join("\n       ")]
         exit 1
@@ -36,18 +36,7 @@ module MCollective
     def self.list
       load_config
 
-      applist = []
-
-      Config.instance.libdir.each do |libdir|
-        applicationdir = "#{libdir}/mcollective/application"
-        next unless File.directory?(applicationdir)
-
-        Dir.entries(applicationdir).grep(/\.rb$/).each do |application|
-          applist << File.basename(application, ".rb")
-        end
-      end
-
-      applist
+      PluginManager.find("application")
     rescue SystemExit
       exit 1
     rescue Exception => e

data/lib/mcollective/cache.rb

@@ -0,0 +1,145 @@
+module MCollective
+  # A class to manage a number of named caches.  Each cache can have a unique
+  # timeout for keys in it and each cache has an independent Mutex protecting
+  # access to it.
+  #
+  # This cache is setup early in the process of loading the mcollective
+  # libraries, before any threads are created etc making it suitable as a
+  # cross thread cache or just a store for Mutexes you need to use across
+  # threads like in an Agent or something.
+  #
+  #    # sets up a new cache, noop if it already exist
+  #    Cache.setup(:ddl, 600)
+  #    => true
+  #
+  #    # writes an item to the :ddl cache, this item will
+  #    # be valid on the cache for 600 seconds
+  #    Cache.write(:ddl, :something, "value")
+  #    => "value"
+  #
+  #    # reads from the cache, read failures due to non existing
+  #    # data or expired data will raise an exception
+  #    Cache.read(:ddl, :something)
+  #    => "value"
+  #
+  #    # time left till expiry, raises if nothing is found
+  #    Cache.ttl(:ddl, :something)
+  #    => 500
+  #
+  #    # forcibly evict something from the cache
+  #    Cache.invalidate!(:ddl, :something)
+  #    => "value"
+  #
+  #    # deletes an entire named cache and its mutexes
+  #    Cache.delete!(:ddl)
+  #    => true
+  #
+  #    # you can also just use this cache as a global mutex store
+  #    Cache.setup(:mylock)
+  #
+  #    Cache.synchronize(:mylock) do
+  #      do_something()
+  #    end
+  #
+  module Cache
+    # protects access to @cache_locks and top level @cache
+    @locks_mutex = Mutex.new
+
+    # stores a mutex per named cache
+    @cache_locks = {}
+
+    # the named caches protected by items in @cache_locks
+    @cache = {}
+
+    def self.setup(cache_name, ttl=300)
+      @locks_mutex.synchronize do
+        break if @cache_locks.include?(cache_name)
+
+        @cache_locks[cache_name] = Mutex.new
+
+        @cache_locks[cache_name].synchronize do
+          @cache[cache_name] = {:max_age => Float(ttl)}
+        end
+      end
+
+      true
+    end
+
+    def self.has_cache?(cache_name)
+      @locks_mutex.synchronize do
+        @cache.include?(cache_name)
+      end
+    end
+
+    def self.delete!(cache_name)
+      @locks_mutex.synchronize do
+        raise("No cache called '%s'" % cache_name) unless @cache_locks.include?(cache_name)
+
+        @cache_locks.delete(cache_name)
+        @cache.delete(cache_name)
+      end
+
+      true
+    end
+
+    def self.write(cache_name, key, value)
+      raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
+
+      @cache_locks[cache_name].synchronize do
+        @cache[cache_name][key] ||= {}
+        @cache[cache_name][key][:cache_create_time] = Time.now
+        @cache[cache_name][key][:value] = value
+      end
+
+      value
+    end
+
+    def self.read(cache_name, key)
+      raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
+
+      unless ttl(cache_name, key) > 0
+        Log.debug("Cache expired on '%s' key '%s'" % [cache_name, key])
+        raise("Cache for item '%s' on cache '%s' has expired" % [key, cache_name])
+      end
+
+      Log.debug("Cache hit on '%s' key '%s'" % [cache_name, key])
+
+      @cache_locks[cache_name].synchronize do
+        @cache[cache_name][key][:value]
+      end
+    end
+
+    def self.ttl(cache_name, key)
+      raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
+
+      @cache_locks[cache_name].synchronize do
+        unless @cache[cache_name].include?(key)
+          Log.debug("Cache miss on '%s' key '%s'" % [cache_name, key])
+          raise("No item called '%s' for cache '%s'" % [key, cache_name])
+        end
+
+        @cache[cache_name][:max_age] - (Time.now - @cache[cache_name][key][:cache_create_time])
+      end
+    end
+
+    def self.synchronize(cache_name)
+      raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
+
+      raise ("No block supplied to synchronize") unless block_given?
+
+      @cache_locks[cache_name].synchronize do
+        yield
+      end
+    end
+
+    def self.invalidate!(cache_name, key)
+      raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
+
+      @cache_locks[cache_name].synchronize do
+        return false unless @cache[cache_name].include?(key)
+
+        @cache[cache_name].delete(key)
+      end
+    end
+  end
+end