mcollective-client 2.9.1 → 2.10.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZWEwMDY2OTlmMTkwOWM4NGRjODVmYzQzMTk1ZGJmN2M4MDA0MTdlZg==
+  data.tar.gz: !binary |-
+    MjAyNjQ5NDQxZTE1MmM1NzEwODcwMDU1MTQzNGVlNmNhYjdkZDcyMw==
+SHA512:
+  metadata.gz: !binary |-
+    NjNjZWZiZjIzMTk5ODE1ZTNkZDVmYThhMGNmNzFhNzY4ZGViMGJiNWU0OGU3
+    MDcyMTlhNjEyZDI3ZGZiMjE4YzVjNzk2OTVkOGY2ZDdlZjg5MTFjNTcwOWUx
+    N2YzZDU4YTg1MThhZWEwZTVhYjQ4NTk1ZDNlZWU2ZTc0ODRhOWQ=
+  data.tar.gz: !binary |-
+    Mjc1NjBiNjJjNWMyYjljNDE0OWFiZmE3OWYwOGNmODEzZmE2ZmIxNWJhNDdk
+    NzA0MmQ5ZmI3Y2JjYmI5ODIwMGE4NzcwMjNmZTdkOWJhYmRkOGJjZWVjN2Vk
+    Njg2YzZiNGI2NThjYWQ2MjA4ZTA3NDUyNDA0M2FhOTgxYmI3MGE=

data/lib/mcollective.rb

@@ -59,7 +59,7 @@ module MCollective
 
   MCollective::Vendor.load_vendored
 
-  VERSION="2.9.1"
+  VERSION="2.10.0"
 
   def self.version
     VERSION

data/lib/mcollective/application/find.rb

@@ -6,6 +6,8 @@ class MCollective::Application::Find<MCollective::Application
 
     starttime = Time.now
 
+    mc.detect_and_set_stdin_discovery
+
     nodes = mc.discover
 
     discoverytime = Time.now - starttime

data/lib/mcollective/application/rpc.rb

@@ -96,18 +96,8 @@ class MCollective::Application::Rpc<MCollective::Application
       puts "Request sent with id: " + mc.send(configuration[:action], configuration[:arguments])
     else
       discover_args = {:verbose => true}
-      # IF the discovery method hasn't been explicitly overridden
-      #  and we're not being run interactively,
-      #  and someone has piped us some data
-      # Then we assume it's a discovery list - this can be either:
-      #  - list of hosts in plaintext
-      #  - JSON that came from another rpc or printrpc
-      if mc.default_discovery_method && !STDIN.tty? && !STDIN.eof?
-          # Then we override discovery to try to grok the data on STDIN
-          mc.discovery_method = 'stdin'
-          mc.discovery_options = 'auto'
-          discover_args = {:verbose => false}
-      end
+
+      mc.detect_and_set_stdin_discovery
 
       mc.discover discover_args
 

data/lib/mcollective/client.rb

@@ -159,10 +159,10 @@ module MCollective
     #
     # It returns a hash of times and timeouts for discovery and total run is taken from the options
     # hash which in turn is generally built using MCollective::Optionparser
-    def req(body, agent=nil, options=false, waitfor=0, &block)
+    def req(body, agent=nil, options=false, waitfor=[], &block)
       if body.is_a?(Message)
         agent = body.agent
-        waitfor = body.discovered_hosts.size || 0
+        waitfor = body.discovered_hosts || []
         @options = body.options
       end
 
@@ -170,12 +170,11 @@ module MCollective
       threaded = @options[:threaded]
       timeout = @discoverer.discovery_timeout(@options[:timeout], @options[:filter])
       request = createreq(body, agent, @options[:filter])
-      publish_timeout = @options[:publish_timeout]
+      publish_timeout = @options[:publish_timeout] || @config.publish_timeout
       stat = {:starttime => Time.now.to_f, :discoverytime => 0, :blocktime => 0, :totaltime => 0}
       STDOUT.sync = true
       hosts_responded = 0
 
-
       begin
         if threaded
           hosts_responded = threaded_req(request, publish_timeout, timeout, waitfor, &block)
@@ -238,16 +237,41 @@ module MCollective
     def start_receiver(requestid, waitfor, timeout, &block)
       Log.debug("Starting response receiver with timeout of #{timeout}")
       hosts_responded = 0
+
+      if (waitfor.is_a?(Array))
+        unfinished = Hash.new(0)
+        waitfor.each {|w| unfinished[w] += 1}
+      else
+        unfinished = []
+      end
+
       begin
         Timeout.timeout(timeout) do
-          begin
+          loop do
             resp = receive(requestid)
             yield resp.payload
             hosts_responded += 1
-          end while (waitfor == 0 || hosts_responded < waitfor)
+
+            if (waitfor.is_a?(Array))
+              sender = resp.payload[:senderid]
+              if unfinished[sender] <= 1
+                unfinished.delete(sender)
+              else
+                unfinished[sender] -= 1
+              end
+
+              break if !waitfor.empty? && unfinished.empty?
+            else
+              break unless waitfor == 0 || hosts_responded < waitfor
+            end
+          end
         end
       rescue Timeout::Error => e
-        if (waitfor > hosts_responded)
+        if waitfor.is_a?(Array)
+          if !unfinished.empty?
+            Log.warn("Could not receive all responses. Did not receive responses from #{unfinished.keys.join(', ')}")
+          end
+        elsif (waitfor > hosts_responded)
           Log.warn("Could not receive all responses. Expected : #{waitfor}. Received : #{hosts_responded}")
         end
       end

data/lib/mcollective/connector/activemq.rb

@@ -541,6 +541,7 @@ module MCollective
         raise("Unknown target type #{type}") unless [:directed, :broadcast, :reply, :request, :direct_request].include?(type)
         raise("Unknown collective '#{collective}' known collectives are '#{@config.collectives.join ', '}'") unless @config.collectives.include?(collective)
 
+        agents_multiplex = get_bool_option("activemq.agents_multiplex", "false")
         target = {:name => nil, :headers => {}}
 
         case type
@@ -548,10 +549,18 @@ module MCollective
             target[:name] = ["/queue/" + collective, :reply, "#{Config.instance.identity}_#{$$}", Client.request_sequence].join(".")
 
           when :broadcast
-            target[:name] = ["/topic/" + collective, agent, :agent].join(".")
+            if agents_multiplex
+              target[:name] = ["/topic/" + collective, :agents].join(".")
+            else
+              target[:name] = ["/topic/" + collective, agent, :agent].join(".")
+            end
 
           when :request
-            target[:name] = ["/topic/" + collective, agent, :agent].join(".")
+            if agents_multiplex
+              target[:name] = ["/topic/" + collective, :agents].join(".")
+            else
+              target[:name] = ["/topic/" + collective, agent, :agent].join(".")
+            end
 
           when :direct_request
             target[:name] = ["/queue/" + collective, :nodes].join(".")

data/lib/mcollective/connector/rabbitmq.rb

@@ -387,6 +387,7 @@ module MCollective
         raise("Unknown target type #{type}") unless [:directed, :broadcast, :reply, :request, :direct_request].include?(type)
         raise("Unknown collective '#{collective}' known collectives are '#{@config.collectives.join ', '}'") unless @config.collectives.include?(collective)
 
+        agents_multiplex = get_bool_option("rabbitmq.agents_multiplex", "false")
         target = {:name => "", :headers => {}, :id => nil}
 
         if reply_to
@@ -402,13 +403,18 @@ module MCollective
             target[:id] = "mcollective_%s_replies" % agent
 
           when :broadcast, :request # publishing a request to all nodes with an agent
-            target[:name] = "/exchange/%s_broadcast/%s" % [collective, agent]
+            if agents_multiplex
+              target[:name] = "/exchange/%s_broadcast" % collective
+              target[:id] = "%s_broadcast" % collective
+            else
+              target[:name] = "/exchange/%s_broadcast/%s" % [collective, agent]
+              target[:id] = "%s_broadcast_%s" % [collective, agent]
+            end
             if reply_to
               target[:headers]["reply-to"] = reply_to
             else
               target[:headers]["reply-to"] = reply_path
             end
-            target[:id] = "%s_broadcast_%s" % [collective, agent]
 
           when :direct_request # a request to a specific node
             raise "Directed requests need to have a node identity" unless node

data/lib/mcollective/discovery/stdin.rb

@@ -18,7 +18,7 @@ module MCollective
         file = STDIN.read
 
         if file =~ /^\s*$/
-              raise("data piped on STDIN contained only whitespace - could not discover hosts from it.")
+          raise("data piped on STDIN contained only whitespace - could not discover hosts from it.")
         end
 
         if type == 'auto'
@@ -29,8 +29,10 @@ module MCollective
           end
         end
 
+        Log.debug("Parsing STDIN input as type %s" % type)
+
         if type == 'json'
-          hosts = MCollective::RPC::Helpers.extract_hosts_from_json(file)
+          hosts = RPC::Helpers.extract_hosts_from_json(file)
         elsif type == 'text'
           hosts = file.split("\n")
         else

data/lib/mcollective/rpc/client.rb

@@ -119,6 +119,12 @@ module MCollective
           @stdout = STDOUT
           @stdout.sync = true
         end
+
+        if initial_options[:stdin]
+          @stdin = initial_options[:stdin]
+        else
+          @stdin = STDIN
+        end
       end
 
       # Disconnects cleanly from the middleware
@@ -459,6 +465,24 @@ module MCollective
         agent_filter @agent
       end
 
+      # Detects data on STDIN and sets the STDIN discovery method
+      #
+      # IF the discovery method hasn't been explicitly overridden
+      #  and we're not being run interactively,
+      #  and someone has piped us some data
+      #
+      # Then we assume it's a discovery list - this can be either:
+      #  - list of hosts in plaintext
+      #  - JSON that came from another rpc or printrpc
+      #
+      # Then we override discovery to try to grok the data on STDIN
+      def detect_and_set_stdin_discovery
+        if self.default_discovery_method && !@stdin.tty? && !@stdin.eof?
+          self.discovery_method = 'stdin'
+          self.discovery_options = 'auto'
+        end
+      end
+
       # Does discovery based on the filters set, if a discovery was
       # previously done return that else do a new discovery.
       #
@@ -845,6 +869,7 @@ module MCollective
             end
 
             @stats.noresponsefrom.concat @client.stats[:noresponsefrom]
+            @stats.unexpectedresponsefrom.concat @client.stats[:unexpectedresponsefrom]
             @stats.responses += @client.stats[:responses]
             @stats.blocktime += @client.stats[:blocktime] + sleep_time
             @stats.totaltime += @client.stats[:totaltime]

data/lib/mcollective/rpc/helpers.rb

@@ -2,8 +2,8 @@ module MCollective
   module RPC
     # Various utilities for the RPC system
     class Helpers
-      # Parse JSON output as produced by printrpc and extract
-      # the "sender" of each rpc response
+      # Parse JSON output as produced by printrpc or puppet query
+      # and extract the "sender" / "certname" of each entry
       #
       # The simplist valid JSON based data would be:
       #
@@ -11,6 +11,13 @@ module MCollective
       #  {"sender" => "example.com"},
       #  {"sender" => "another.com"}
       # ]
+      #
+      # or
+      #
+      # [
+      #  {"certname" => "example.com"},
+      #  {"certname" => "another.com"}
+      # ]
       def self.extract_hosts_from_json(json)
         hosts = JSON.parse(json)
 
@@ -18,9 +25,12 @@ module MCollective
 
         hosts.map do |host|
           raise "JSON host list is not an array of Hashes" unless host.is_a?(Hash)
-          raise "JSON host list does not have senders in it" unless host.include?("sender")
 
-          host["sender"]
+          unless host.include?("sender") || host.include?("certname")
+            raise "JSON host list does not have senders in it"
+          end
+
+          host["sender"] || host["certname"]
         end.uniq
       end
 
@@ -264,7 +274,7 @@ module MCollective
         parser.on('--one', '-1', 'Send request to only one discovered nodes') do |v|
           options[:mcollective_limit_targets] = 1
         end
-    
+
         parser.on('--batch SIZE', 'Do requests in batches') do |v|
           # validate batch string. Is it x% where x > 0 or is it an integer
           if ((v =~ /^(\d+)%$/ && Integer($1) != 0) || v =~ /^(\d+)$/)

data/lib/mcollective/rpc/stats.rb

@@ -2,9 +2,9 @@ module MCollective
   module RPC
     # Class to wrap all the stats and to keep track of some timings
     class Stats
-      attr_accessor :noresponsefrom, :starttime, :discoverytime, :blocktime, :responses, :totaltime
-      attr_accessor :discovered, :discovered_nodes, :okcount, :failcount, :noresponsefrom, :responsesfrom
-      attr_accessor :requestid, :aggregate_summary, :ddl, :aggregate_failures
+      attr_accessor :noresponsefrom, :unexpectedresponsefrom, :starttime, :discoverytime, :blocktime, :responses
+      attr_accessor :totaltime, :discovered, :discovered_nodes, :okcount, :failcount, :noresponsefrom
+      attr_accessor :responsesfrom, :requestid, :aggregate_summary, :ddl, :aggregate_failures
 
       def initialize
         reset
@@ -13,6 +13,7 @@ module MCollective
       # Resets stats, if discovery time is set we keep it as it was
       def reset
         @noresponsefrom = []
+        @unexpectedresponsefrom = []
         @responsesfrom = []
         @responses = 0
         @starttime = Time.now.to_f
@@ -23,7 +24,6 @@ module MCollective
         @discovered_nodes = []
         @okcount = 0
         @failcount = 0
-        @noresponsefrom = []
         @requestid = nil
         @aggregate_summary = []
         @aggregate_failures = []
@@ -32,6 +32,7 @@ module MCollective
       # returns a hash of our stats
       def to_hash
         {:noresponsefrom    => @noresponsefrom,
+         :unexpectedresponsefrom => @unexpectedresponsefrom,
          :starttime         => @starttime,
          :discoverytime     => @discoverytime,
          :blocktime         => @blocktime,
@@ -70,6 +71,7 @@ module MCollective
       # Re-initializes the object with stats from the basic client
       def client_stats=(stats)
         @noresponsefrom = stats[:noresponsefrom]
+        @unexpectedresponsefrom = stats[:unexpectedresponsefrom]
         @responses = stats[:responses]
         @starttime = stats[:starttime]
         @blocktime = stats[:blocktime]
@@ -115,13 +117,19 @@ module MCollective
       def finish_request
         @totaltime = @blocktime + @discoverytime
 
-        # figures out who we had no responses from
+        # figure out who responded unexpectedly
+        rhosts = @responsesfrom.clone
+        @discovered_nodes.each {|d| rhosts.delete(d)}
+        @unexpectedresponsefrom = rhosts
+
+        # figure out who we had no responses from
         dhosts = @discovered_nodes.clone
         @responsesfrom.each {|r| dhosts.delete(r)}
         @noresponsefrom = dhosts
       rescue
         @totaltime = 0
         @noresponsefrom = []
+        @unexpectedresponsefrom = []
       end
 
       # Helper to keep track of who we received responses from
@@ -225,8 +233,22 @@ module MCollective
           end
         end
 
-        if no_response_report != ""
-          result_text << "" << no_response_report
+        no_response_r = no_response_report
+        unexpected_response_r = unexpected_response_report
+        if no_response_r || unexpected_response_r
+          result_text << ""
+        end
+
+        if no_response_r != ""
+          result_text << "" << no_response_r
+        end
+
+        if unexpected_response_r != ""
+          result_text << "" << unexpected_response_r
+        end
+
+        if no_response_r || unexpected_response_r
+          result_text << ""
         end
 
         result_text.join("\n")
@@ -237,7 +259,6 @@ module MCollective
         result_text = StringIO.new
 
         if @noresponsefrom.size > 0
-          result_text.puts
           result_text.puts Util.colorize(:red, "No response from:")
           result_text.puts
 
@@ -248,8 +269,26 @@ module MCollective
           @noresponsefrom.sort.in_groups_of(fields_num) do |c|
             result_text.puts format % c
           end
+        end
 
+        result_text.string
+      end
+
+      # Returns a blob of text indicating what nodes responded but weren't discovered
+      def unexpected_response_report
+        result_text = StringIO.new
+
+        if @unexpectedresponsefrom.size > 0
+          result_text.puts Util.colorize(:red, "Unexpected response from:")
           result_text.puts
+
+          field_size = Util.field_size(@unexpectedresponsefrom, 30)
+          fields_num = Util.field_number(field_size)
+          format = "   " + ( " %-#{field_size}s" * fields_num )
+
+          @unexpectedresponsefrom.sort.in_groups_of(fields_num) do |c|
+            result_text.puts format % c
+          end
         end
 
         result_text.string

data/spec/unit/mcollective/client_spec.rb

@@ -205,14 +205,25 @@ module MCollective
 
       it "should thread the publisher and receiver if configured" do
         client.instance_variable_get(:@options)[:threaded] = true
-        client.expects(:threaded_req).with(request, nil, 0, 1)
+        client.expects(:threaded_req).with(request, 2, 0, ['rspec'])
         message.options[:threaded] = true
         client.req(message)
       end
 
       it "should not thread the publisher and receiver if configured" do
         client.instance_variable_set(:@threaded, false)
-        client.expects(:unthreaded_req).with(request, nil, 0, 1)
+        client.expects(:unthreaded_req).with(request, 2, 0, ['rspec'])
+        client.req(message)
+      end
+
+      it "uses the publish_timeout from options when passed as an option" do
+        client.expects(:unthreaded_req).with(request, 5, 0, ['rspec'])
+        client.req(message, nil, message.options.merge(:publish_timeout => 5))
+      end
+
+      it "uses the publish_timeout from config when passed as a config value" do
+        client.expects(:unthreaded_req).with(request, 10, 0, ['rspec'])
+        client.instance_variable_get(:@config).expects(:publish_timeout).returns(10)
         client.req(message)
       end
     end
@@ -267,50 +278,132 @@ module MCollective
     end
 
     describe "#start_receiver" do
-      it "should go into a receive loop and receive until it reaches waitfor" do
-        results = []
-        Timeout.stubs(:timeout).yields
-        message = mock
-        client.stubs(:receive).with("erfs123").returns(message)
-        message.stubs(:payload).returns("msg1", "msg2", "msg3")
-        client.start_receiver("erfs123", 3, 5) do |msg|
-          results << msg
+      describe "waitfor is a number" do
+        it "should go into a receive loop and receive until it reaches waitfor" do
+          results = []
+          Timeout.stubs(:timeout).yields
+          message = mock
+          client.stubs(:receive).with("erfs123").returns(message)
+          message.stubs(:payload).returns("msg1", "msg2", "msg3")
+          client.start_receiver("erfs123", 3, 5) do |msg|
+            results << msg
+          end
+          results.should == ["msg1", "msg2", "msg3"]
         end
-        results.should == ["msg1", "msg2", "msg3"]
-      end
 
-      it "should log a warning if a timeout occurs" do
-        results = []
-        Timeout.stubs(:timeout).yields
-        message = mock
-        client.stubs(:receive).with("erfs123").returns(message)
-        message.stubs(:payload).returns("msg1", "msg2", "timeout")
-        Log.expects(:warn).with("Could not receive all responses. Expected : 3. Received : 2")
-        responded = client.start_receiver("erfs123", 3, 5) do |msg|
-          if msg == "timeout"
-            raise Timeout::Error
+        it "should log a warning if a timeout occurs" do
+          results = []
+          Timeout.stubs(:timeout).yields
+          message = mock
+          client.stubs(:receive).with("erfs123").returns(message)
+          message.stubs(:payload).returns("msg1", "msg2", "timeout")
+          Log.expects(:warn).with("Could not receive all responses. Expected : 3. Received : 2")
+          responded = client.start_receiver("erfs123", 3, 5) do |msg|
+            if msg == "timeout"
+              raise Timeout::Error
+            end
+            results << msg
+          end
+          results.should == ["msg1", "msg2"]
+          responded.should == 2
+        end
+
+        it "should not log a warning if a the response count is larger or equal to the expected number of responses" do
+          results = []
+          Timeout.stubs(:timeout).yields
+          message = mock
+          client.stubs(:receive).with("erfs123").returns(message)
+          message.stubs(:payload).returns("msg1", "msg2", "timeout")
+          Log.expects(:warn).never
+          responded = client.start_receiver("erfs123", 2, 5) do |msg|
+            if msg == "timeout"
+              raise Timeout::Error
+            end
+            results << msg
           end
-          results << msg
+          results.should == ["msg1", "msg2"]
+          responded.should == 2
         end
-        results.should == ["msg1", "msg2"]
-        responded.should == 2
       end
 
-      it "should not log a warning if a the response count is larger or equal to the expected number of responses" do
-        results = []
-        Timeout.stubs(:timeout).yields
-        message = mock
-        client.stubs(:receive).with("erfs123").returns(message)
-        message.stubs(:payload).returns("msg1", "msg2", "timeout")
-        Log.expects(:warn).never
-        responded = client.start_receiver("erfs123", 2, 5) do |msg|
-          if msg == "timeout"
-            raise Timeout::Error
+      describe "waitfor is an array" do
+        it "should go into a receive loop and receive until it matches waitfor" do
+          senders = ["sender1", "sender2", "sender3", "sender4"]
+          expected = senders.map {|s| Message.new({:callerid => "caller", :senderid => s}, nil, :type => :reply)}
+          results = []
+          Timeout.stubs(:timeout).yields
+          client.stubs(:receive).with("erfs123").returns(*expected)
+          client.start_receiver("erfs123", senders[0,3], 5) do |msg|
+            results << msg
+          end
+          results.should == expected[0,3].map {|m| m.payload}
+        end
+
+        it "receive until it gets all expected responses" do
+          senders = ["sender1", "sender2", "sender3", "sender4"]
+          expected = senders.map {|s| Message.new({:callerid => "caller", :senderid => s}, nil, :type => :reply)}
+          results = []
+          Timeout.stubs(:timeout).yields
+          client.stubs(:receive).with("erfs123").returns(*expected)
+          client.start_receiver("erfs123", senders[1,3], 5) do |msg|
+            results << msg
+          end
+          results.should == expected.map {|m| m.payload}
+        end
+
+        it "should log a warning if a timeout occurs" do
+          senders = ["sender1", "sender2", "sender3"]
+          messages = ["msg1", "msg2", "timeout"]
+          expected = senders.zip(messages).map {|s, m| Message.new({:callerid => "caller", :senderid => s, :body => m}, nil, :type => :reply)}
+          results = []
+          Timeout.stubs(:timeout).yields
+          client.stubs(:receive).with("erfs123").returns(*expected)
+          Log.expects(:warn).with("Could not receive all responses. Did not receive responses from sender3")
+          responded = client.start_receiver("erfs123", senders, 5) do |msg|
+            if msg[:body] == "timeout"
+              raise Timeout::Error
+            end
+            results << msg
+          end
+          results.should == expected[0,2].map {|m| m.payload}
+          responded.should == 2
+        end
+
+        it "should not log a warning if accepting all responses" do
+          senders = ["sender1", "sender2", "sender3"]
+          messages = ["msg1", "msg2", "timeout"]
+          expected = senders.zip(messages).map {|s, m| Message.new({:callerid => "caller", :senderid => s, :body => m}, nil, :type => :reply)}
+          results = []
+          Timeout.stubs(:timeout).yields
+          client.stubs(:receive).with("erfs123").returns(*expected)
+          Log.expects(:warn).never
+          responded = client.start_receiver("erfs123", [], 1) do |msg|
+            if msg[:body] == "timeout"
+              raise Timeout::Error
+            end
+            results << msg
+          end
+          results.should == expected[0,2].map {|m| m.payload}
+          responded.should == 2
+        end
+
+        it "should not log a warning if the response count is larger or equal to the expected number of responses" do
+          senders = ["sender1", "sender2", "sender3"]
+          messages = ["msg1", "msg2", "timeout"]
+          expected = senders.zip(messages).map {|s, m| Message.new({:callerid => "caller", :senderid => s, :body => m}, nil, :type => :reply)}
+          results = []
+          Timeout.stubs(:timeout).yields
+          client.stubs(:receive).with("erfs123").returns(*expected)
+          Log.expects(:warn).never
+          responded = client.start_receiver("erfs123", senders[0,2], 5) do |msg|
+            if msg[:body] == "timeout"
+              raise Timeout::Error
+            end
+            results << msg
           end
-          results << msg
+          results.should == expected[0,2].map {|m| m.payload}
+          responded.should == 2
         end
-        results.should == ["msg1", "msg2"]
-        responded.should == 2
       end
     end