right_agent 0.14.0 → 0.16.2

data/lib/right_agent/packets.rb

@@ -104,7 +104,8 @@ module RightScale
       @size = msg.size
       # For ruby 1.9 size attribute moves from front to back of packet
       re = RUBY_VERSION < "1.9.0" ? /size\xC0/ : /size\xC0$/
-      msg.sub!(re) { |m| "size" + @size.to_msgpack }
+      # For msgpack 0.5.1 the to_msgpack result is a MessagePack::Packer so need to convert to string
+      msg = msg.to_s.sub!(re) { |m| "size" + @size.to_msgpack }
       msg
     end
 

data/lib/right_agent/platform/darwin.rb

@@ -102,6 +102,12 @@ module RightScale
         '/var/spool'
       end
 
+      def ssh_cfg_dir
+        # TODO This is a guess, but since we don't have Darwin instances
+        # it may need to be corrected later.
+        '/etc/ssh'
+      end
+
       # Cached data from applications. Such data is locally generated as a
       # result of time-consuming I/O or calculation. The application must
       # be able to regenerate or restore the data.

data/lib/right_agent/platform/linux.rb

@@ -153,6 +153,10 @@ module RightScale
         '/var/spool'
       end
 
+      def ssh_cfg_dir
+        '/etc/ssh'
+      end
+
       # Cached data from applications. Such data is locally generated as a
       # result of time-consuming I/O or calculation. The application must
       # be able to regenerate or restore the data.
@@ -262,7 +266,7 @@ module RightScale
       # ParserError:: on failure to parse volume list
       def parse_volumes(output_text, conditions = nil)
         results = []
-        output_text.each do |line|
+        output_text.lines.each do |line|
           volume = {}
           line_regex = /^([\/a-z0-9_\-\.]+):(.*)/
           volmatch = line_regex.match(line)

data/lib/right_agent/platform/windows.rb

@@ -175,6 +175,10 @@ module RightScale
         return pretty_path(File.join(Dir::COMMON_APPDATA, 'RightScale', 'spool'))
       end
 
+      def ssh_cfg_dir
+        return pretty_path(File.join(ENV['USERPROFILE'] || temp_dir, '.ssh'))
+      end
+
       # Cache directory for the current platform
       def cache_dir
         return pretty_path(File.join(Dir::COMMON_APPDATA, 'RightScale', 'cache'))
@@ -656,7 +660,7 @@ EOF
         line_regex = nil
         header_regex = /  --------  (-+)  -------  -------  ---  ---/
         header_match = nil
-        output_text.each do |line|
+        output_text.lines.each do |line|
           line = line.chomp
           if line_regex
             if line.strip.empty?
@@ -722,7 +726,7 @@ EOF
         header_regex = /  ----------  ---  (-+)  (-+)  (-+)  -------  (-+)  (-+)/
         header_match = nil
         line_regex = nil
-        output_text.each do |line|
+        output_text.lines.each do |line|
           line = line.chomp
           if line_regex
             if line.strip.empty?
@@ -805,7 +809,7 @@ EOF
         header_regex = /  -------------  (-+)  -------  -------/
         header_match = nil
         line_regex = nil
-        output_text.each do |line|
+        output_text.lines.each do |line|
           line = line.chomp
           if line_regex
             if line.strip.empty?
@@ -1113,7 +1117,7 @@ EOF
           # the $Error list if necessary), so this is a catch-all for any
           # script which does not handle errors "properly".
           lines_after_script << "if ($NULL -eq $LastExitCode) { $LastExitCode = 0 }"
-          lines_after_script << "if ((0 -eq $LastExitCode) -and ($Error.Count -gt 0)) { $RS_message = 'Script exited successfully but $Error contained '+($Error.Count)+' error(s).'; Write-warning $RS_message; $LastExitCode = 1 }"
+          lines_after_script << "if ((0 -eq $LastExitCode) -and ($Error.Count -gt 0)) { $RS_message = 'Script exited successfully but $Error contained '+($Error.Count)+' error(s):'; write-output $RS_message; write-output $Error; $LastExitCode = 1 }"
         end
 
         # ensure last exit code gets marshalled.

data/lib/right_agent/scripts/stats_manager.rb

@@ -22,7 +22,7 @@
 #
 #    Options:
 #      --reset, -r        As part of gathering the stats from an agent also reset the stats
-#      --timeout, -t SEC  Override default timeout in seconds to wait for a response from an agent
+#      --timeout, -T SEC  Override default timeout in seconds to wait for a response from an agent
 #      --json, -j         Display the stats data in JSON format
 #      --verbose, -v      Log debug information
 #      --cfg-dir, -c DIR  Set directory containing configuration for all agents
@@ -86,7 +86,7 @@ module RightScale
           options[:reset] = true
         end
 
-        opts.on('-t', '--timeout SEC') do |sec|
+        opts.on('-T', '--timeout SEC') do |sec|
           options[:timeout] = sec
         end
 
@@ -162,7 +162,7 @@ module RightScale
         client = CommandClient.new(listen_port, config_options[:cookie])
         command = {:name => :stats, :reset => options[:reset]}
         begin
-          client.send_command(command, options[:verbose], options[:timeout]) { |r| display(agent_name, r, options) }
+          client.send_command(command, verbose = false, options[:timeout]) { |r| display(agent_name, r, options) }
           res = true
         rescue Exception => e
           msg = "Could not retrieve #{agent_name} agent stats: #{e}"

data/lib/right_agent/security/cached_certificate_store_proxy.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2009-2011 RightScale Inc
+# Copyright (c) 2009-2013 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -28,34 +28,48 @@ module RightScale
     # Initialize cache proxy with given certificate store
     #
     # === Parameters
-    # store(Object):: Certificate store responding to get_recipients and
-    #   get_signer
+    # store(Object):: Certificate store responding to get_signer, get_target,
+    # and get_receiver
     def initialize(store)
       @signer_cache = CertificateCache.new
       @store = store
     end
 
-    # Retrieve recipient certificates
+    # Retrieve signer certificates for use in verifying a signature
+    # Check cache first and cache results
+    #
+    # === Parameters
+    # id(String):: Serialized identity of signer
+    #
+    # === Return
+    # (Array|Certificate):: Signer certificate(s)
+    def get_signer(id)
+      @signer_cache.get(id) { @store.get_signer(id) }
+    end
+
+    # Retrieve certificates of target for encryption
     # Results are not cached
     #
     # === Parameters
-    # packet(RightScale::Packet):: Packet containing recipient identity, ignored
+    # packet(RightScale::Packet):: Packet containing target identity
     #
     # === Return
-    # (Array):: Recipient certificates
-    def get_recipients(obj)
-      @store.get_recipients(obj)
+    # (Array|Certificate):: Target certificate(s)
+    def get_target(obj)
+      @store.get_target(obj)
     end
 
-    # Check cache for signer certificate
+    # Retrieve receiver's certificate and key for decryption
+    # Results are not cached
     #
     # === Parameters
-    # id(String):: Serialized identity of signer
+    # id(String|nil):: Optional identifier of source of data for use
+    #   in determining who is the receiver
     #
     # === Return
-    # (Array):: Signer certificates
-    def get_signer(id)
-      @signer_cache.get(id) { @store.get_signer(id) }
+    # (Array):: Certificate and key
+    def get_receiver(id)
+      @store.get_receiver(id)
     end
 
   end # CachedCertificateStoreProxy

data/lib/right_agent/security/encrypted_document.rb

@@ -33,8 +33,7 @@ module RightScale
     #
     # === Parameters
     # data(String):: Data to be encrypted
-    # certs(Array):: Recipient certificates (certificates corresponding to private
-    #   keys that may be used to decrypt data)
+    # certs(Array|Certificate):: Target recipient certificates used to encrypt data
     # cipher(Cipher):: Cipher used for encryption, AES 256 CBC by default
     def initialize(data, certs, cipher = 'AES-256-CBC')
       cipher = OpenSSL::Cipher::Cipher.new(cipher)

data/lib/right_agent/security/static_certificate_store.rb

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2009-2011 RightScale Inc
+# Copyright (c) 2009-2013 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -22,46 +22,62 @@
 
 module RightScale
 
-  # Simple certificate store, serves a static set of certificates
+  # Simple certificate store that serves a static set of certificates and one key
   class StaticCertificateStore
     
     # Initialize store
     #
     # === Parameters
+    # receiver_cert(Certificate):: Certificate for decrypting serialized data being received
+    # receiver_key(RsaKeyPair):: Key corresponding to specified cert
     # signer_certs(Array|Certificate):: Signer certificate(s) used when loading data to
     #   check the digital signature. The signature associated with the serialized data
     #   needs to match with one of the signer certificates for loading to succeed.
-    # recipients_certs(Array|Certificate):: Recipient certificate(s) used when serializing
+    # target_certs(Array|Certificate):: Target certificate(s) used when serializing
     #   data for encryption. Loading the data can only be done through serializers that
-    #   have been initialized with a certificate that's in the recipient certificates
+    #   have been initialized with a certificate that's in the target certificates
     #   if encryption is enabled.
-    def initialize(signer_certs, recipients_certs)
+    def initialize(receiver_cert, receiver_key, signer_certs, target_certs)
+      @receiver_cert = receiver_cert
+      @receiver_key = receiver_key
       signer_certs = [ signer_certs ] unless signer_certs.respond_to?(:each)
       @signer_certs = signer_certs 
-      recipients_certs = [ recipients_certs ] unless recipients_certs.respond_to?(:each)
-      @recipients_certs = recipients_certs
+      target_certs = [ target_certs ] unless target_certs.respond_to?(:each)
+      @target_certs = target_certs
     end
     
-    # Retrieve signer certificates
+    # Retrieve signer certificates for use in verifying a signature
     #
     # === Parameters
     # id(String):: Serialized identity of signer, ignored
     #
     # === Return
-    # (Array):: Signer certificates
+    # (Array|Certificate):: Signer certificates
     def get_signer(id)
       @signer_certs
     end
 
-    # Retrieve recipient certificates that will be able to decrypt the serialized data
+    # Retrieve certificates of target for encryption
     #
     # === Parameters
-    # packet(RightScale::Packet):: Packet containing recipient identity, ignored
+    # packet(RightScale::Packet):: Packet containing target identity, ignored
     #
     # === Return
-    # (Array):: Recipient certificates
-    def get_recipients(packet)
-      @recipients_certs
+    # (Array|Certificate):: Target certificates
+    def get_target(packet)
+      @target_certs
+    end
+
+    # Retrieve receiver's certificate and key for decryption
+    #
+    # === Parameters
+    # id(String|nil):: Optional identifier of source of data for use
+    #   in determining who is the receiver, ignored
+    #
+    # === Return
+    # (Array):: Certificate and key
+    def get_receiver(id)
+      [@receiver_cert, @receiver_key]
     end
     
   end # StaticCertificateStore

data/lib/right_agent/sender.rb

@@ -46,11 +46,15 @@ module RightScale
       # (String) Token for parent request in a retry situation
       attr_accessor :retry_parent
 
+      # (String) Non-delivery reason if any
+      attr_accessor :non_delivery
+
       def initialize(kind, receive_time, response_handler)
         @kind = kind
         @receive_time = receive_time
         @response_handler = response_handler
         @retry_parent = nil
+        @non_delivery = nil
       end
 
     end # PendingRequest
@@ -808,7 +812,7 @@ module RightScale
       build_and_send_packet(:send_persistent_request, type, payload, target, callback)
     end
 
-    # Build packet
+    # Build packet or queue it if offline
     #
     # === Parameters
     # kind(Symbol):: Kind of send request: :send_push, :send_persistent_push, :send_retryable_request,
@@ -825,30 +829,69 @@ module RightScale
     # callback(Boolean):: Whether this request has an associated response callback
     #
     # === Return
-    # (Push|Request):: Packet created
+    # (Push|Request|NilClass):: Packet created, or nil if queued instead
+    #
+    # === Raise
+    # ArgumentError:: If target is invalid
     def build_packet(kind, type, payload, target, callback = false)
-      kind_str = kind.to_s
-      persistent = !!(kind_str =~ /persistent/)
-      if kind_str =~ /push/
-        packet = Push.new(type, payload)
-        packet.selector = target[:selector] || :any if target.is_a?(Hash)
-        packet.confirm = true if callback
+      validate_target(target, !!(kind.to_s =~ /push/))
+      if should_queue?
+        @offline_handler.queue_request(kind, type, payload, target, callback)
+        nil
       else
-        packet = Request.new(type, payload)
-        ttl = @options[:time_to_live]
-        packet.expires_at = Time.now.to_i + ttl if !persistent && ttl && ttl != 0
-        packet.selector = :any
+        kind_str = kind.to_s
+        persistent = !!(kind_str =~ /persistent/)
+        if kind_str =~ /push/
+          packet = Push.new(type, payload)
+          packet.selector = target[:selector] || :any if target.is_a?(Hash)
+          packet.confirm = true if callback
+        else
+          packet = Request.new(type, payload)
+          ttl = @options[:time_to_live]
+          packet.expires_at = Time.now.to_i + ttl if !persistent && ttl && ttl != 0
+          packet.selector = :any
+        end
+        packet.from = @identity
+        packet.token = AgentIdentity.generate
+        packet.persistent = persistent
+        if target.is_a?(Hash)
+          packet.tags = target[:tags] || []
+          packet.scope = target[:scope]
+        else
+          packet.target = target
+        end
+        packet
       end
-      packet.from = @identity
-      packet.token = AgentIdentity.generate
-      packet.persistent = persistent
-      if target.is_a?(Hash)
-        packet.tags = target[:tags] || []
-        packet.scope = target[:scope]
-      else
-        packet.target = target
+    end
+
+    # Send packet
+    #
+    # === Parameters
+    # kind(Symbol):: Kind of send request: :send_push, :send_persistent_push, :send_retryable_request,
+    #   or :send_persistent_request
+    # packet(Push|Request|NilClass):: Packet to send; nothing sent if nil
+    # callback(Proc|nil):: Block used to process routing response
+    #
+    # === Return
+    # true:: Always return true
+    #
+    # === Raise
+    # SendFailure:: If publishing of request fails unexpectedly
+    # TemporarilyOffline:: If cannot publish request because currently not connected
+    #    to any brokers and offline queueing is disabled
+    def send_packet(kind, packet, callback)
+      if packet
+        method = packet.type.split('/').last
+        received_at = @request_stats.update(method, packet.token)
+        @request_kind_stats.update((packet.selector == :all ? kind.to_s.sub(/push/, "fanout") : kind.to_s)[5..-1])
+        @pending_requests[packet.token] = PendingRequest.new(kind, received_at, callback) if callback
+        if !packet.persistent && kind.to_s =~ /request/
+          publish_with_timeout_retry(packet, packet.token)
+        else
+          publish(packet)
+        end
       end
-      packet
+      true
     end
 
     # Handle response to a request
@@ -873,11 +916,23 @@ module RightScale
         end
 
         if handler = @pending_requests[token]
-          if result && result.non_delivery? && handler.kind == :send_retryable_request &&
-             [OperationResult::TARGET_NOT_CONNECTED, OperationResult::TTL_EXPIRATION].include?(result.content)
-            # Log and ignore so that timeout retry mechanism continues
-            # Leave purging of associated request until final response, i.e., success response or retry timeout
-            Log.info("Non-delivery of <#{token}> because #{result.content}")
+          if result && result.non_delivery? && handler.kind == :send_retryable_request
+            if [OperationResult::TARGET_NOT_CONNECTED, OperationResult::TTL_EXPIRATION].include?(result.content)
+              # Log and temporarily ignore so that timeout retry mechanism continues, but save reason for use below if timeout
+              # Leave purging of associated request until final response, i.e., success response or retry timeout
+              if parent = handler.retry_parent
+                @pending_requests[parent].non_delivery = result.content
+              else
+                handler.non_delivery = result.content
+              end
+              Log.info("Non-delivery of <#{token}> because #{result.content}")
+            elsif result.content == OperationResult::RETRY_TIMEOUT && handler.non_delivery
+              # Request timed out but due to another non-delivery reason, so use that reason since more germane
+              response.results = OperationResult.non_delivery(handler.non_delivery)
+              deliver(response, handler)
+            else
+              deliver(response, handler)
+            end
           else
             deliver(response, handler)
           end
@@ -1085,21 +1140,8 @@ module RightScale
     # TemporarilyOffline:: If cannot publish request because currently not connected
     #    to any brokers and offline queueing is disabled
     def build_and_send_packet(kind, type, payload, target, callback)
-      validate_target(target, allow_selector = !!(kind.to_s =~ /push/))
-      if should_queue?
-        @offline_handler.queue_request(kind, type, payload, target, callback)
-      else
-        packet = build_packet(kind, type, payload, target, callback)
-        method = type.split('/').last
-        received_at = @request_stats.update(method, packet.token)
-        @request_kind_stats.update((packet.selector == :all ? kind.to_s.sub(/push/, "fanout") : kind.to_s)[5..-1])
-        @pending_requests[packet.token] = PendingRequest.new(kind, received_at, callback) if callback
-        if !packet.persistent && kind.to_s =~ /request/
-          publish_with_timeout_retry(packet, packet.token)
-        else
-          publish(packet)
-        end
-      end
+      packet = build_packet(kind, type, payload, target, callback)
+      send_packet(kind, packet, callback)
       true
     end
 
@@ -1123,12 +1165,12 @@ module RightScale
         @broker.publish(exchange, request, :persistent => request.persistent, :mandatory => true,
                         :log_filter => [:tags, :target, :tries, :persistent], :brokers => ids)
       rescue RightAMQP::HABrokerClient::NoConnectedBrokers => e
-        msg = "Failed to publish request #{request.to_s([:tags, :target, :tries])}"
+        msg = "Failed to publish request #{request.trace} #{request.type}"
         Log.error(msg, e)
         @send_failure_stats.update("NoConnectedBrokers")
         raise TemporarilyOffline.new(msg + " (#{e.class}: #{e.message})")
       rescue Exception => e
-        msg = "Failed to publish request #{request.to_s([:tags, :target, :tries])}"
+        msg = "Failed to publish request #{request.trace} #{request.type}"
         Log.error(msg, e, :trace)
         @send_failure_stats.update(e.class.name)
         @exception_stats.track("publish", e, request)
@@ -1156,7 +1198,7 @@ module RightScale
     def publish_with_timeout_retry(request, parent, count = 0, multiplier = 1, elapsed = 0, broker_ids = nil)
       published_broker_ids = publish(request, broker_ids)
 
-      if @retry_interval && @retry_timeout && parent && !published_broker_ids.empty?
+      if @retry_interval && @retry_timeout && parent
         interval = [(@retry_interval * multiplier) + (@request_stats.avg_duration || 0), @retry_timeout - elapsed].min
         EM.add_timer(interval) do
           begin
@@ -1173,15 +1215,27 @@ module RightScale
                                            broker_ids.push(broker_ids.shift))
                 @retry_stats.update(request.type.split('/').last)
               else
-                Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{request.to_s([:tags, :target, :tries])}")
+                Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{request.trace} #{request.type}")
                 result = OperationResult.non_delivery(OperationResult::RETRY_TIMEOUT)
                 @non_delivery_stats.update(result.content)
                 handle_response(Result.new(request.token, request.reply_to, result, @identity))
               end
-              @connectivity_checker.check(published_broker_ids.first) if count == 1
+              @connectivity_checker.check(published_broker_ids.first) if count == 1 && !published_broker_ids.empty?
             end
+          rescue TemporarilyOffline => e
+            # Send retry response so that requester, e.g., IdempotentRequest, can retry
+            Log.error("Failed retry for #{request.trace} #{request.type} because temporarily offline")
+            result = OperationResult.retry("lost connectivity")
+            handle_response(Result.new(request.token, request.reply_to, result, @identity))
+          rescue SendFailure => e
+            # Send non-delivery response so that requester, e.g., IdempotentRequest, can retry
+            Log.error("Failed retry for #{request.trace} #{request.type} because of send failure")
+            result = OperationResult.non_delivery("retry failed")
+            handle_response(Result.new(request.token, request.reply_to, result, @identity))
           rescue Exception => e
-            Log.error("Failed retry for #{request.token}", e, :trace)
+            # Not sending a response here because something more basic is broken in the retry
+            # mechanism and don't want an error response to preempt a delayed actual response
+            Log.error("Failed retry for #{request.trace} #{request.type} without responding", e, :trace)
             @exception_stats.track("retry", e, request)
           end
         end