mcollective-client 2.0.0 → 2.2.0

data/lib/mcollective/message.rb

@@ -19,6 +19,7 @@ module MCollective
     # options[:options]        - the normal client options hash
     # options[:ttl]            - the maximum amount of seconds this message can be valid for
     # options[:expected_msgid] - in the case of replies this is the msgid it is expecting in the replies
+    # options[:requestid]      - specific request id to use else one will be generated
     def initialize(payload, message, options = {})
       options = {:base64 => false,
                  :agent => nil,
@@ -29,11 +30,12 @@ module MCollective
                  :options => {},
                  :ttl => 60,
                  :expected_msgid => nil,
+                 :requestid => nil,
                  :collective => nil}.merge(options)
 
       @payload = payload
       @message = message
-      @requestid = nil
+      @requestid = options[:requestid]
       @discovered_hosts = nil
       @reply_to = nil
 
@@ -72,15 +74,21 @@ module MCollective
     # this is to force a workflow that doesnt not yield in a mistake when someone might assume
     # direct_addressing is enabled when its not.
     def type=(type)
+      raise "Unknown message type #{type}" unless VALIDTYPES.include?(type)
+
       if type == :direct_request
         raise "Direct requests is not enabled using the direct_addressing config option" unless Config.instance.direct_addressing
 
         unless @discovered_hosts && !@discovered_hosts.empty?
           raise "Can only set type to :direct_request if discovered_hosts have been set"
         end
-      end
 
-      raise "Unknown message type #{type}" unless VALIDTYPES.include?(type)
+        # clear out the filter, custom discovery sources might interpret the filters
+        # different than the remote mcollectived and in directed mode really the only
+        # filter that matters is the agent filter
+        @filter = Util.empty_filter
+        @filter["agent"] << @agent
+      end
 
       @type = type
     end
@@ -131,13 +139,43 @@ module MCollective
           @requestid = request.payload[:requestid]
           @payload = PluginManager["security_plugin"].encodereply(agent, payload, requestid, request.payload[:callerid])
         when :request, :direct_request
-          @requestid = create_reqid
+          validate_compount_filter(@filter["compound"]) unless @filter["compound"].empty?
+
+          @requestid = create_reqid unless @requestid
           @payload = PluginManager["security_plugin"].encoderequest(Config.instance.identity, payload, requestid, filter, agent, collective, ttl)
         else
           raise "Cannot encode #{type} messages"
       end
     end
 
+    def validate_compount_filter(compound_filter)
+      compound_filter.each do |filter|
+        filter.each do |statement|
+          if statement["fstatement"]
+            functionname = statement["fstatement"]["name"]
+            pluginname = Data.pluginname(functionname)
+            value = statement["fstatement"]["value"]
+
+            begin
+              ddl = DDL.new(pluginname, :data)
+            rescue
+              raise DDLValidationError, "Could not find DDL for data plugin #{pluginname}, cannot use #{functionname}() in discovery"
+            end
+
+            # parses numbers and booleans entered as strings into proper
+            # types of data so that DDL validation will pass
+            statement["fstatement"]["params"] = Data.ddl_transform_input(ddl, statement["fstatement"]["params"])
+
+            Data.ddl_validate(ddl, statement["fstatement"]["params"])
+
+            unless value && Data.ddl_has_output?(ddl, value)
+              raise DDLValidationError, "#{functionname}() does not return a #{value} value"
+            end
+          end
+        end
+      end
+    end
+
     def decode!
       raise "Cannot decode message type #{type}" unless [:request, :reply].include?(type)
 
@@ -183,7 +221,7 @@ module MCollective
         # send it as is.
         if @discovered_hosts && Config.instance.direct_addressing
           if @discovered_hosts.size <= Config.instance.direct_addressing_threshold
-            @type = :direct_request
+            self.type = :direct_request
             Log.debug("Handling #{requestid} as a direct request")
           end
 
@@ -195,7 +233,10 @@ module MCollective
     end
 
     def create_reqid
-      Digest::MD5.hexdigest("#{Config.instance.identity}-#{Time.now.to_f}-#{agent}-#{collective}")
+      # we gsub out the -s so that the format of the id does not
+      # change from previous versions, these should just be more
+      # unique than previous ones
+      SSL.uuid.gsub("-", "")
     end
   end
 end

data/lib/mcollective/monkey_patches.rb

@@ -1,3 +1,11 @@
+# start_with? was introduced in 1.8.7, we need to support
+# 1.8.5 and 1.8.6
+class String
+  def start_with?(str)
+    return self[0 .. (str.length-1)] == str
+  end unless method_defined?("start_with?")
+end
+
 # Make arrays of Symbols sortable
 class Symbol
   include Comparable
@@ -51,6 +59,15 @@ class Array
   end unless method_defined?(:in_groups_of)
 end
 
+class String
+  def bytes(&block)
+    # This should not be necessary, really ...
+    require 'enumerator'
+    return to_enum(:each_byte) unless block_given?
+    each_byte(&block)
+  end unless method_defined?(:bytes)
+end
+
 class Dir
   def self.mktmpdir(prefix_suffix=nil, tmpdir=nil)
     case prefix_suffix

data/lib/mcollective/optionparser.rb

@@ -86,7 +86,7 @@ module MCollective
       end
 
       @parser.on('-S', '--select FILTER', 'Compound filter combining facts and classes') do |f|
-        @options[:filter]["compound"] << MCollective::Matcher::Parser.new(f).execution_stack
+        @options[:filter]["compound"] << Matcher.create_compound_callstack(f)
       end
 
       @parser.on('-F', '--wf', '--with-fact fact=val', 'Match hosts with a certain fact') do |f|
@@ -152,6 +152,23 @@ module MCollective
       @parser.on('--reply-to TARGET', 'Set a custom target for replies') do |v|
         @options[:reply_to] = v
       end
+
+      @parser.on('--dm', '--disc-method METHOD', 'Which discovery method to use') do |v|
+        raise "Discovery method is already set by a competing option" if @options[:discovery_method] && @options[:discovery_method] != v
+        @options[:discovery_method] = v
+      end
+
+      @parser.on('--do', '--disc-option OPTION', 'Options to pass to the discovery method') do |a|
+        @options[:discovery_options] << a
+      end
+
+      @parser.on("--nodes FILE", "List of nodes to address") do |v|
+        raise "Cannot mix --disc-method, --disc-option and --nodes" if @options[:discovery_method] || @options[:discovery_options].size > 0
+        raise "Cannot read the discovery file #{v}" unless File.readable?(v)
+
+        @options[:discovery_method] = "flatfile"
+        @options[:discovery_options] = v
+      end
     end
 
     private

data/lib/mcollective/pluginmanager.rb

@@ -60,7 +60,7 @@ module MCollective
 
     # Provides a list of plugins we know about
     def self.pluginlist
-      @plugins.keys
+      @plugins.keys.sort
     end
 
     # deletes all registered plugins
@@ -119,7 +119,7 @@ module MCollective
       plugins = []
 
       Config.instance.libdir.each do |libdir|
-        plugdir = File.join([libdir, "mcollective", type])
+        plugdir = File.join([libdir, "mcollective", type.to_s])
         next unless File.directory?(plugdir)
 
         Dir.new(plugdir).grep(/#{extension}$/).map do |plugin|
@@ -174,7 +174,7 @@ module MCollective
 
     # Grep's over the plugin list and returns the list found
     def self.grep(regex)
-      @plugins.keys.grep regex
+      @plugins.keys.grep(regex)
     end
   end
 end

data/lib/mcollective/pluginpackager.rb

@@ -15,9 +15,16 @@ module MCollective
 
     # Fetch and return metadata from plugin DDL
     def self.get_metadata(path, type)
-      ddl = MCollective::RPC::DDL.new("package", false)
-      ddl.instance_eval File.read(Dir.glob(File.join(path, type, "*.ddl")).first)
-      ddl.meta
+      ddl = DDL.new("package", type.to_sym, false)
+
+      begin
+        ddl_file = File.read(Dir.glob(File.join(path, type, "*.ddl")).first)
+      rescue Exception
+        raise "failed to load ddl file in plugin directory : #{File.join(path, type)}"
+      end
+      ddl.instance_eval ddl_file
+
+      return ddl.meta, ddl.requirements[:mcollective]
     end
 
     # Checks if a directory is present and not empty

data/lib/mcollective/pluginpackager/agent_definition.rb

@@ -3,9 +3,9 @@ module MCollective
     # MCollective Agent Plugin package
     class AgentDefinition
       attr_accessor :path, :packagedata, :metadata, :target_path, :vendor, :iteration, :preinstall
-      attr_accessor :plugintype, :dependencies, :postinstall, :mcserver, :mcclient, :mccommon
+      attr_accessor :plugintype, :dependencies, :postinstall, :mcname, :mcversion
 
-      def initialize(path, name, vendor, preinstall, postinstall, iteration, dependencies, mcodependency, plugintype)
+      def initialize(path, name, vendor, preinstall, postinstall, iteration, dependencies, mcdependency, plugintype)
         @plugintype = plugintype
         @path = path
         @packagedata = {}
@@ -13,12 +13,13 @@ module MCollective
         @preinstall = preinstall
         @postinstall = postinstall
         @vendor = vendor || "Puppet Labs"
-        @mcserver = mcodependency[:server] || "mcollective"
-        @mcclient = mcodependency[:client] || "mcollective-client"
-        @mccommon = mcodependency[:common] || "mcollective-common"
         @dependencies = dependencies || []
         @target_path = File.expand_path(@path)
-        @metadata = PluginPackager.get_metadata(@path, "agent")
+        @metadata, mcversion = PluginPackager.get_metadata(@path, "agent")
+        @mcname = mcdependency[:mcname] ||  "mcollective"
+        @mcversion = mcdependency[:mcversion] || mcversion
+        @dependencies << {:name => "#{@mcname}-common", :version => @mcversion}
+
         @metadata[:name] = (name || @metadata[:name]).downcase.gsub(" ", "-")
         identify_packages
       end
@@ -36,7 +37,7 @@ module MCollective
       # Obtain Agent package files and dependencies.
       def agent
         agent = {:files => [],
-                 :dependencies => @dependencies.clone << @mcserver,
+                 :dependencies => @dependencies.clone,
                  :description => "Agent plugin for #{@metadata[:name]}"}
 
         agentdir = File.join(@path, "agent")
@@ -44,43 +45,50 @@ module MCollective
         if PluginPackager.check_dir_present agentdir
           ddls = Dir.glob(File.join(agentdir, "*.ddl"))
           agent[:files] = (Dir.glob(File.join(agentdir, "*")) - ddls)
-          implementations = Dir.glob(File.join(@metadata[:name], "**"))
-          agent[:files] += implementations unless implementations.empty?
         else
           return nil
         end
-        agent[:dependencies] << "mcollective-#{@metadata[:name]}-common" if @packagedata[:common]
+        agent[:dependencies] << {:name => "#{@mcname}-#{@metadata[:name]}-common", :version => @metadata[:version]}
         agent
       end
 
       # Obtain client package files and dependencies.
       def client
         client = {:files => [],
-                  :dependencies => @dependencies.clone << @mcclient,
+                  :dependencies => @dependencies.clone,
                   :description => "Client plugin for #{@metadata[:name]}"}
 
         clientdir = File.join(@path, "application")
-        bindir = File.join(@path, "bin")
-        ddldir = File.join(@path, "agent")
+        aggregatedir = File.join(@path, "aggregate")
 
         client[:files] += Dir.glob(File.join(clientdir, "*")) if PluginPackager.check_dir_present clientdir
-        client[:files] += Dir.glob(File.join(bindir,"*")) if PluginPackager.check_dir_present bindir
-        client[:files] += Dir.glob(File.join(ddldir, "*.ddl")) if PluginPackager.check_dir_present ddldir
-        client[:dependencies] << "mcollective-#{@metadata[:name]}-common" if @packagedata[:common]
+        client[:files] += Dir.glob(File.join(aggregatedir, "*")) if PluginPackager.check_dir_present aggregatedir
+        client[:dependencies] << {:name => "#{@mcname}-#{@metadata[:name]}-common", :version => @metadata[:version]}
         client[:files].empty? ? nil : client
       end
 
       # Obtain common package files and dependencies.
       def common
         common = {:files =>[],
-                  :dependencies => @dependencies.clone << @mccommon,
+                  :dependencies => @dependencies.clone,
                   :description => "Common libraries for #{@metadata[:name]}"}
 
-        commondir = File.join(@path, "util")
-        common[:files] += Dir.glob(File.join(commondir,"*")) if PluginPackager.check_dir_present commondir
+        datadir = File.join(@path, "data", "**")
+        utildir = File.join(@path, "util", "**", "**")
+        ddldir = File.join(@path, "agent", "*.ddl")
+        validatordir = File.join(@path, "validator", "**")
+
+        [datadir, utildir, validatordir, ddldir].each do |directory|
+          common[:files] += Dir.glob(directory)
+        end
+
+        # We fail if there is no ddl file present
+        if common[:files].grep(/^.*\.ddl$/).empty?
+          raise "cannot create package - No ddl file found in #{File.join(@path, "agent")}"
+        end
+
         common[:files].empty? ? nil : common
       end
-
     end
   end
 end

data/lib/mcollective/pluginpackager/standard_definition.rb

@@ -2,10 +2,9 @@ module MCollective
   module PluginPackager
     class StandardDefinition
       attr_accessor :path, :packagedata, :metadata, :target_path, :vendor, :iteration
-      attr_accessor :plugintype, :preinstall, :postinstall, :dependencies, :mcserver
-      attr_accessor :mccommon
+      attr_accessor :plugintype, :preinstall, :postinstall, :dependencies, :mcname, :mcversion
 
-      def initialize(path, name, vendor, preinstall, postinstall, iteration, dependencies, mcodependency, plugintype)
+      def initialize(path, name, vendor, preinstall, postinstall, iteration, dependencies, mcdependency, plugintype)
         @plugintype = plugintype
         @path = path
         @packagedata = {}
@@ -14,10 +13,12 @@ module MCollective
         @postinstall = postinstall
         @vendor = vendor || "Puppet Labs"
         @dependencies = dependencies || []
-        @mcserver = mcodependency[:server] || "mcollective"
-        @mccommon = mcodependency[:common] || "mcollective-common"
         @target_path = File.expand_path(@path)
-        @metadata = PluginPackager.get_metadata(@path, @plugintype)
+        @metadata, mcversion = PluginPackager.get_metadata(@path, @plugintype)
+
+        @mcname = mcdependency[:mcname] || "mcollective"
+        @mcversion = mcdependency[:mcversion] || mcversion
+        @dependencies << {:name => "#{mcname}-common", :version => @mcversion}
         @metadata[:name] = (name || @metadata[:name]).downcase.gsub(" ", "-")
         identify_packages
       end
@@ -33,7 +34,7 @@ module MCollective
       # Obtain standard plugin files and dependencies
       def plugin
         plugindata = {:files => [],
-                      :dependencies => @dependencies.clone << @mcserver,
+                      :dependencies => @dependencies.clone,
                       :description => "#{@name} #{@plugintype} plugin for the Marionette Collective."}
 
         plugindir = File.join(@path, @plugintype.to_s)
@@ -43,14 +44,15 @@ module MCollective
           return nil
         end
 
-        plugindata[:dependencies] <<"mcollective-#{@metadata[:name]}-common" if @packagedata[:common]
+        plugindata[:dependencies] << {:name => "#{@mcname}-#{@metadata[:name]}-common",
+                                      :version => @metadata[:version]} if @packagedata[:common]
         plugindata
       end
 
       # Obtain list of common files
       def common
         common = {:files => [],
-                  :dependencies => @dependencies.clone << @mccommon,
+                  :dependencies => @dependencies.clone,
                   :description => "Common libraries for #{@name} connector plugin"}
 
         commondir = File.join(@path, "util")

data/lib/mcollective/registration/base.rb

@@ -38,7 +38,9 @@ module MCollective
       end
 
       def msg_filter
-        {"agent" => "registration"}
+        filter = Util.empty_filter
+        filter["agent"] << "registration"
+        filter
       end
 
       def target_collective

data/lib/mcollective/rpc.rb

@@ -5,17 +5,16 @@ module MCollective
   # an RPC metaphor, standard compliant agents will make it easier
   # to create generic clients like web interfaces etc
   module RPC
-    autoload :Client, "mcollective/rpc/client"
+    autoload :ActionRunner, "mcollective/rpc/actionrunner"
     autoload :Agent, "mcollective/rpc/agent"
-    autoload :Reply, "mcollective/rpc/reply"
-    autoload :Request, "mcollective/rpc/request"
     autoload :Audit, "mcollective/rpc/audit"
+    autoload :Client, "mcollective/rpc/client"
+    autoload :Helpers, "mcollective/rpc/helpers"
     autoload :Progress, "mcollective/rpc/progress"
-    autoload :Stats, "mcollective/rpc/stats"
-    autoload :DDL, "mcollective/rpc/ddl"
+    autoload :Reply, "mcollective/rpc/reply"
+    autoload :Request, "mcollective/rpc/request"
     autoload :Result, "mcollective/rpc/result"
-    autoload :Helpers, "mcollective/rpc/helpers"
-    autoload :ActionRunner, "mcollective/rpc/actionrunner"
+    autoload :Stats, "mcollective/rpc/stats"
 
     # Creates a standard options hash, pass in a block to add extra headings etc
     # see Optionparser
@@ -115,18 +114,18 @@ module MCollective
     # If you've passed -v on the command line a detailed stat block
     # will be printed, else just a one liner.
     #
-    # You can pass flags into it, at the moment only one flag is
-    # supported:
+    # You can pass flags into it:
     #
-    # printrpcstats :caption => "Foo"
+    #   printrpcstats :caption => "Foo", :summarize => true
     #
     # This will use "Foo" as the caption to the stats in verbose
-    # mode
+    # mode and print out any aggregate summary information if present
     def printrpcstats(flags={})
       return unless @options[:output_format] == :console
 
+      flags = {:summarize => false, :caption => "rpc stats"}.merge(flags)
+
       verbose = @options[:verbose] rescue verbose = false
-      caption = flags[:caption] || "rpc stats"
 
       begin
         stats = @@stats
@@ -135,8 +134,7 @@ module MCollective
         return
       end
 
-      puts
-      puts stats.report(caption, verbose)
+      puts stats.report(flags[:caption], flags[:summarize], verbose)
     end
 
     # Prints the result of an RPC call.
@@ -150,8 +148,9 @@ module MCollective
       verbose = flags[:verbose] || verbose
       flatten = flags[:flatten] || false
       format = @options[:output_format]
+      forced_mode = @options[:force_display_mode] || false
 
-      result_text =  Helpers.rpcresults(result, {:verbose => verbose, :flatten => flatten, :format => format})
+      result_text =  Helpers.rpcresults(result, {:verbose => verbose, :flatten => flatten, :format => format, :force_display_mode => forced_mode})
 
       if result.is_a?(Array) && format == :console
         puts "\n%s\n" % [ result_text ]
@@ -173,16 +172,11 @@ module MCollective
       end
     end
 
-    # Factory for RPC::Request messages, only really here to make agents
-    # a bit easier to understand
-    def self.request(msg)
-      RPC::Request.new(msg)
-    end
+    def self.const_missing(const_name)
+      super unless const_name == :DDL
 
-    # Factory for RPC::Reply messages, only really here to make agents
-    # a bit easier to understand
-    def self.reply
-      RPC::Reply.new
+      Log.warn("MCollective::RPC::DDL is deprecatd, please use MCollective::DDL instead")
+      MCollective::DDL
     end
   end
 end