right_agent 2.1.5 → 2.2.0

data/lib/right_agent/agent_tag_manager.rb

@@ -98,6 +98,8 @@ module RightScale
     #
     # === Parameters
     # new_tags(String, Array):: Tag or tags to be added
+    # options(Hash):: Request options
+    #   :timeout(Integer):: timeout in seconds before giving up and yielding an error message
     #
     # === Block
     # A block is optional. If provided, should take one argument which will be set with the
@@ -105,15 +107,17 @@ module RightScale
     #
     # === Return
     # true always return true
-    def add_tags(new_tags)
+    def add_tags(new_tags, options = {})
       new_tags = ensure_flat_array_value(new_tags) unless new_tags.nil? || new_tags.empty?
-      do_update(new_tags, []) { |raw_response| yield raw_response if block_given? }
+      do_update(new_tags, [], options) { |raw_response| yield raw_response if block_given? }
     end
 
     # Remove given tags from agent
     #
     # === Parameters
     # old_tags(String, Array):: Tag or tags to be removed
+    # options(Hash):: Request options
+    #   :timeout(Integer):: timeout in seconds before giving up and yielding an error message
     #
     # === Block
     # A block is optional. If provided, should take one argument which will be set with the
@@ -121,20 +125,24 @@ module RightScale
     #
     # === Return
     # true always return true
-    def remove_tags(old_tags)
+    def remove_tags(old_tags, options = {})
       old_tags = ensure_flat_array_value(old_tags) unless old_tags.nil? || old_tags.empty?
-      do_update([], old_tags) { |raw_response| yield raw_response if block_given? }
+      do_update([], old_tags, options) { |raw_response| yield raw_response if block_given? }
     end
 
     # Clear all agent tags
     #
+    # === Parameters
+    # options(Hash):: Request options
+    #   :timeout(Integer):: timeout in seconds before giving up and yielding an error message
+    #
     # === Block
     # Given block should take one argument which will be set with the raw response
     #
     # === Return
     # true::Always return true
-    def clear
-      do_update([], @agent.tags) { |raw_response| yield raw_response }
+    def clear(options = {})
+      do_update([], @agent.tags, options) { |raw_response| yield raw_response }
     end
 
     private
@@ -197,7 +205,7 @@ module RightScale
     #
     # === Return
     # true:: Always return true
-    def do_update(new_tags, old_tags, &block)
+    def do_update(new_tags, old_tags, options = {}, &block)
       agent_check
       raise ArgumentError.new("Cannot add and remove tags in same update") if new_tags.any? && old_tags.any?
       tags = @agent.tags
@@ -206,9 +214,9 @@ module RightScale
       tags.uniq!
 
       if new_tags.any?
-        request = RightScale::RetryableRequest.new("/router/add_tags", {:tags => new_tags})
+        request = RightScale::RetryableRequest.new("/router/add_tags", {:tags => new_tags}, options)
       elsif old_tags.any?
-        request = RightScale::RetryableRequest.new("/router/delete_tags", {:tags => old_tags})
+        request = RightScale::RetryableRequest.new("/router/delete_tags", {:tags => old_tags}, options)
       else
         return
       end

data/lib/right_agent/clients/api_client.rb

@@ -109,8 +109,11 @@ module RightScale
     #       Packet::GLOBAL, ones with no shard id
     #   [Symbol] :selector for picking from qualified targets: :any or :all;
     #     defaults to :any
-    # @param [String, NilClass] token uniquely identifying this request;
-    #   defaults to randomly generated ID
+    #
+    # @option options [String] :request_uuid uniquely identifying this request; defaults to
+    #   randomly generated
+    # @option options [Numeric] :time_to_live seconds before request expires and is to be ignored;
+    #   non-positive value or nil means never expire
     #
     # @return [NilClass] always nil since there is no expected response to the request
     #
@@ -120,8 +123,8 @@ module RightScale
     # @raise [Exceptions::RetryableError] request failed but if retried may succeed
     # @raise [Exceptions::Terminating] closing client and terminating service
     # @raise [Exceptions::InternalServerError] internal error in server being accessed
-    def push(type, payload, target, token = nil)
-      map_request(type, payload, token)
+    def push(type, payload, target, options = {})
+      map_request(type, payload, options)
     end
 
     # Route a request to a single target with a response expected
@@ -140,8 +143,11 @@ module RightScale
     #   [Array] :tags that must all be associated with a target for it to be selected
     #   [Hash] :scope for restricting routing which may contain:
     #     [Integer] :account id that agents must be associated with to be included
-    # @param [String, NilClass] token uniquely identifying this request;
-    #   defaults to randomly generated ID
+    #
+    # @option options [String] :request_uuid uniquely identifying this request; defaults to
+    #   randomly generated
+    # @option options [Numeric] :time_to_live seconds before request expires and is to be ignored;
+    #   non-positive value or nil means never expire
     #
     # @return [Result, NilClass] response from request
     #
@@ -151,8 +157,8 @@ module RightScale
     # @raise [Exceptions::RetryableError] request failed but if retried may succeed
     # @raise [Exceptions::Terminating] closing client and terminating service
     # @raise [Exceptions::InternalServerError] internal error in server being accessed
-    def request(type, payload, target, token = nil)
-      map_request(type, payload, token)
+    def request(type, payload, target, options = {})
+      map_request(type, payload, options)
     end
 
     # Determine whether request supported by this client
@@ -170,8 +176,9 @@ module RightScale
     #
     # @param [String] type of request as path specifying actor and action
     # @param [Hash, NilClass] payload for request
-    # @param [String, NilClass] token uniquely identifying this request;
-    #   defaults to randomly generated ID
+    #
+    # @option options [String] :request_uuid uniquely identifying this request
+    # @option options [Numeric] :time_to_live seconds before request expires and is to be ignored
     #
     # @return [Object, NilClass] response from request
     #
@@ -181,15 +188,15 @@ module RightScale
     # @raise [Exceptions::RetryableError] request failed but if retried may succeed
     # @raise [Exceptions::Terminating] closing client and terminating service
     # @raise [Exceptions::InternalServerError] internal error in server being accessed
-    def map_request(type, payload, token)
+    def map_request(type, payload, options)
       verb, path = API_MAP[type]
       raise ArgumentError, "Unsupported request type: #{type}" if path.nil?
       actor, action = type.split("/")[1..-1]
-      path, params, options = parameterize(actor, action, payload, path)
+      path, params, request_options = parameterize(actor, action, payload, path)
       if action == "query_tags"
-        map_query_tags(verb, params, action, token, options)
+        map_query_tags(verb, params, action, options.merge(request_options))
       else
-        map_response(make_request(verb, path, params, action, token, options), path)
+        map_response(make_request(verb, path, params, action, options.merge(request_options)), path)
       end
     end
 
@@ -227,16 +234,14 @@ module RightScale
     # @param [Symbol] verb for HTTP REST request
     # @param [Hash] params for HTTP request
     # @param [String] action from request type
-    # @param [String, NilClass] token uniquely identifying this request;
-    #   defaults to randomly generated ID
     # @param [Hash] options augmenting or overriding default options for HTTP request
     #
     # @return [Hash] tags retrieved with resource href as key and tags array as value
-    def map_query_tags(verb, params, action, token, options)
+    def map_query_tags(verb, params, action, options)
       response = {}
       hrefs = params[:resource_hrefs] || []
-      hrefs.concat(query_by_tag(verb, params[:tags], action, token, options)) if params[:tags]
-      response = query_by_resource(verb, hrefs, action, token, options) if hrefs.any?
+      hrefs.concat(query_by_tag(verb, params[:tags], action, options)) if params[:tags]
+      response = query_by_resource(verb, hrefs, action, options) if hrefs.any?
       response
     end
 
@@ -245,15 +250,13 @@ module RightScale
     # @param [Symbol] verb for HTTP REST request
     # @param [Array] tags that all resources retrieved must have
     # @param [String] action from request type
-    # @param [String, NilClass] token uniquely identifying this request;
-    #   defaults to randomly generated ID
     # @param [Hash] options augmenting or overriding default options for HTTP request
     #
     # @return [Array] resource hrefs
-    def query_by_tag(verb, tags, action, token, options)
+    def query_by_tag(verb, tags, action, options)
       path = "/tags/by_tag"
       params = {:tags => tags, :match_all => false, :resource_type => "instances"}
-      map_response(make_request(verb, path, params, action, token, options), path).keys
+      map_response(make_request(verb, path, params, action, options), path).keys
     end
 
     # Query API for tags associated with a set of resources
@@ -261,15 +264,13 @@ module RightScale
     # @param [Symbol] verb for HTTP REST request
     # @param [Array] hrefs for resources whose tags are to be retrieved
     # @param [String] action from request type
-    # @param [String, NilClass] token uniquely identifying this request;
-    #   defaults to randomly generated ID
     # @param [Hash] options augmenting or overriding default options for HTTP request
     #
     # @return [Hash] tags retrieved with resource href as key and tags array as value
-    def query_by_resource(verb, hrefs, action, token, options)
+    def query_by_resource(verb, hrefs, action, options)
       path = "/tags/by_resource"
       params = {:resource_hrefs => hrefs}
-      map_response(make_request(verb, path, params, action, token, options), path)
+      map_response(make_request(verb, path, params, action, options), path)
     end
 
     # Convert payload to HTTP parameters

data/lib/right_agent/clients/balanced_http_client.rb

@@ -37,9 +37,9 @@ module RightScale
     class NotResponding < Exceptions::NestedException; end
 
     # HTTP status codes for which a retry is warranted, which is limited to when server
-    # is not accessible for some reason (502, 503) or server response indicates that
+    # is not accessible for some reason (408, 502, 503) or server response indicates that
     # the request could not be routed for some retryable reason (504)
-    RETRY_STATUS_CODES = [502, 503, 504]
+    RETRY_STATUS_CODES = [408, 502, 503, 504]
 
     # Default time for HTTP connection to open
     DEFAULT_OPEN_TIMEOUT = 2
@@ -183,6 +183,16 @@ module RightScale
       raise
     end
 
+    # Close all persistent connections
+    #
+    # @param [String] reason for closing
+    #
+    # @return [TrueClass] always true
+    def close(reason)
+      @http_client.close(reason) if @http_client
+      true
+    end
+
     protected
 
     # Construct headers for request
@@ -244,9 +254,15 @@ module RightScale
         return [result, code, body, headers] if (Time.now - started_at) >= request_timeout
       end
       if result.nil? && (connection = @http_client.connections[path]) && Time.now < connection[:expires_at]
-        # Continue to poll using same connection until get result, timeout, or hit error
-        used[:host] = connection[:host]
-        result, code, body, headers = @http_client.poll(connection, request_options, started_at + request_timeout)
+        begin
+          # Continue to poll using same connection until get result, timeout, or hit error
+          used[:host] = connection[:host]
+          result, code, body, headers = @http_client.poll(connection, request_options, started_at + request_timeout)
+        rescue HttpException, RestClient::Exception => e
+          raise NotResponding.new(e.http_body, e) if RETRY_STATUS_CODES.include?(e.http_code)
+          raise NotResponding.new("Request timeout", e) if e.is_a?(RestClient::RequestTimeout)
+          raise
+        end
       end
       [result, code, body, headers]
     end
@@ -280,6 +296,10 @@ module RightScale
         else
           raise NotResponding.new("#{server_name} not responding", e)
         end
+      elsif e.is_a?(RestClient::RequestTimeout)
+        # Special case RequestTimeout because http_code is typically nil given no actual response
+        yield(e)
+        raise NotResponding.new("Request timeout", e)
       else
         yield(e)
         raise e

data/lib/right_agent/clients/base_retry_client.rb

@@ -147,6 +147,7 @@ module RightScale
         @reconnect_timer.cancel if @reconnect_timer
         @reconnect_timer = nil
       end
+      close_http_client("terminating")
       true
     end
 
@@ -223,23 +224,36 @@ module RightScale
     end
 
     # Create HTTP client
+    # If there is an existing client, close it first
     #
     # @return [TrueClass] always true
     #
     # @return [BalancedHttpClient] client
     def create_http_client
+      close_http_client("reconnecting")
       url = @auth_client.send(@type.to_s + "_url")
       Log.info("Connecting to #{@options[:server_name]} via #{url.inspect}")
-      options = {
-        :server_name => @options[:server_name],
-        :open_timeout => @options[:open_timeout],
-        :request_timeout => @options[:request_timeout], }
+      options = {:server_name => @options[:server_name]}
       options[:api_version] = @options[:api_version] if @options[:api_version]
       options[:non_blocking] = @options[:non_blocking] if @options[:non_blocking]
       options[:filter_params] = @options[:filter_params] if @options[:filter_params]
       @http_client = RightScale::BalancedHttpClient.new(url, options)
     end
 
+    # Close HTTP client persistent connections
+    #
+    # @param [String] reason for closing
+    #
+    # @return [Boolean] false if failed, otherwise true
+    def close_http_client(reason)
+      @http_client.close(reason) if @http_client
+      true
+    rescue Exception => e
+      Log.error("Failed closing connection", e, :trace)
+      @stats["exceptions"].track("status", e)
+      false
+    end
+
     # Perform any other steps needed to make this client fully usable
     # once HTTP client has been created and server known to be accessible
     #
@@ -303,6 +317,7 @@ module RightScale
     #   - Underlying BalancedHttpClient connection open timeout (:open_timeout)
     #   - Underlying BalancedHttpClient request timeout (:request_timeout)
     #   - Retry timeout for this method and its handlers (:retry_timeout)
+    #   - Seconds before request expires and is to be ignored (:time_to_live)
     # and if the target server is a RightNet router:
     #   - Router response timeout (ideally > :retry_timeout and < :request_timeout)
     #   - Router retry timeout (ideally = :retry_timeout)
@@ -325,6 +340,8 @@ module RightScale
     # @param [String] type of request for use in logging; defaults to path
     # @param [String, NilClass] request_uuid uniquely identifying this request;
     #   defaults to randomly generated UUID
+    # @param [Numeric, NilClass] time_to_live seconds before request expires and is to be ignored;
+    #   non-positive value or nil means never expire; defaults to nil
     # @param [Hash] options augmenting or overriding default options for HTTP request
     #
     # @return [Object, NilClass] result of request with nil meaning no result
@@ -335,10 +352,13 @@ module RightScale
     # @raise [Exceptions::RetryableError] request failed but if retried may succeed
     # @raise [Exceptions::Terminating] closing client and terminating service
     # @raise [Exceptions::InternalServerError] internal error in server being accessed
-    def make_request(verb, path, params = {}, type = nil, request_uuid = nil, options = {})
+    def make_request(verb, path, params = {}, type = nil, options = {})
       raise Exceptions::Terminating if state == :closed
-      request_uuid ||= RightSupport::Data::UUID.generate
       started_at = Time.now
+      time_to_live = (options[:time_to_live] && options[:time_to_live] > 0) ? options[:time_to_live] : nil
+      expires_at = started_at + [time_to_live || @options[:retry_timeout], @options[:retry_timeout]].min
+      headers = time_to_live ? @auth_client.headers.merge("X-Expires-At" => started_at + time_to_live) : @auth_client.headers
+      request_uuid = options[:request_uuid] || RightSupport::Data::UUID.generate
       attempts = 0
       result = nil
       @stats["requests sent"].measure(type || path, request_uuid) do
@@ -348,11 +368,11 @@ module RightScale
             :open_timeout => @options[:open_timeout],
             :request_timeout => @options[:request_timeout],
             :request_uuid => request_uuid,
-            :headers => @auth_client.headers }
+            :headers => headers }
           raise Exceptions::ConnectivityFailure, "#{@type} client not connected" unless [:connected, :closing].include?(state)
           result = @http_client.send(verb, path, params, http_options.merge(options))
         rescue StandardError => e
-          request_uuid = handle_exception(e, type || path, request_uuid, started_at, attempts)
+          request_uuid = handle_exception(e, type || path, request_uuid, expires_at, attempts)
           request_uuid ? retry : raise
         end
       end
@@ -366,17 +386,17 @@ module RightScale
     # @param [String] action from request type
     # @param [String] type of request for use in logging
     # @param [String] request_uuid originally created for this request
-    # @param [Time] started_at time for request
+    # @param [Time] expires_at time for request
     # @param [Integer] attempts to make request
     #
-    # @return [String, NilClass] request token to be used on retry or nil if to raise instead
+    # @return [String, NilClass] request UUID to be used on retry or nil if to raise instead
     #
     # @raise [Exceptions::Unauthorized] authorization failed
     # @raise [Exceptions::ConnectivityFailure] cannot connect to server, lost connection
     #   to it, or it is out of service or too busy to respond
     # @raise [Exceptions::RetryableError] request failed but if retried may succeed
     # @raise [Exceptions::InternalServerError] internal error in server being accessed
-    def handle_exception(exception, type, request_uuid, started_at, attempts)
+    def handle_exception(exception, type, request_uuid, expires_at, attempts)
       result = request_uuid
       if exception.respond_to?(:http_code)
         case exception.http_code
@@ -388,7 +408,7 @@ module RightScale
           @auth_client.expired
           raise Exceptions::RetryableError.new("Authorization expired", exception)
         when 449 # RetryWith
-          result = handle_retry_with(exception, type, request_uuid, started_at, attempts)
+          result = handle_retry_with(exception, type, request_uuid, expires_at, attempts)
         when 500 # InternalServerError
           raise Exceptions::InternalServerError.new(exception.http_body, @options[:server_name])
         else
@@ -396,7 +416,7 @@ module RightScale
           result = nil
         end
       elsif exception.is_a?(BalancedHttpClient::NotResponding)
-        handle_not_responding(exception, type, request_uuid, started_at, attempts)
+        handle_not_responding(exception, type, request_uuid, expires_at, attempts)
       else
         @stats["request failures"].update("#{type} - #{exception.class.name}")
         result = nil
@@ -429,7 +449,7 @@ module RightScale
       true
     end
 
-    # Handle retry response by retrying it once
+    # Handle retry response by retrying it only once
     # This indicates the request was received but a retryable error prevented
     # it from being processed; the retry responsibility may be passed on
     # If retrying, this function does not return until it is time to retry
@@ -437,26 +457,24 @@ module RightScale
     # @param [RestClient::RetryWith] retry_result exception raised
     # @param [String] type of request for use in logging
     # @param [String] request_uuid originally created for this request
-    # @param [Time] started_at time for request
+    # @param [Time] expires_at time for request
     # @param [Integer] attempts to make request
     #
-    # @return [String] request token to be used on retry
+    # @return [String] request UUID to be used on retry
     #
     # @raise [Exceptions::RetryableError] request failed but if retried may succeed
-    def handle_retry_with(retry_result, type, request_uuid, started_at, attempts)
-      if @options[:retry_enabled]
-        interval = @options[:retry_intervals][attempts - 1]
-        if attempts == 1 && interval && (Time.now - started_at) < @options[:retry_timeout]
-          Log.error("Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
-                    "in response to retryable error (#{retry_result.http_body})")
-          wait(interval)
-        else
-          @stats["request failures"].update("#{type} - retry")
-          raise Exceptions::RetryableError.new(retry_result.http_body, retry_result)
-        end
-      else
+    def handle_retry_with(retry_result, type, request_uuid, expires_at, attempts)
+      case (interval = retry_interval(expires_at, attempts, 1))
+      when nil
+        @stats["request failures"].update("#{type} - retry")
+        raise Exceptions::RetryableError.new(retry_result.http_body, retry_result)
+      when 0
         @stats["request failures"].update("#{type} - retry")
         raise Exceptions::RetryableError.new(retry_result.http_body, retry_result)
+      else
+        Log.error("Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
+                  "in response to retryable error (#{retry_result.http_body})")
+        wait(interval)
       end
       # Change request_uuid so that retried request not rejected as duplicate
       "#{request_uuid}:retry"
@@ -469,33 +487,49 @@ module RightScale
     #   indicating targeted server is too busy or out of service
     # @param [String] type of request for use in logging
     # @param [String] request_uuid originally created for this request
-    # @param [Time] started_at time for request
+    # @param [Time] expires_at time for request
     # @param [Integer] attempts to make request
     #
     # @return [TrueClass] always true
     #
     # @raise [Exceptions::ConnectivityFailure] cannot connect to server, lost connection
     #   to it, or it is out of service or too busy to respond
-    def handle_not_responding(not_responding, type, request_uuid, started_at, attempts)
-      if @options[:retry_enabled]
-        interval = @options[:retry_intervals][attempts - 1]
-        if interval && (Time.now - started_at) < @options[:retry_timeout]
-          Log.error("Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
-                    "in response to routing failure (#{BalancedHttpClient.exception_text(not_responding)})")
-          wait(interval)
-        else
-          @stats["request failures"].update("#{type} - no result")
-          self.state = :disconnected
-          raise Exceptions::ConnectivityFailure.new(not_responding.message + " after #{attempts} attempts")
-        end
-      else
+    def handle_not_responding(not_responding, type, request_uuid, expires_at, attempts)
+      case (interval = retry_interval(expires_at, attempts))
+      when nil
         @stats["request failures"].update("#{type} - no result")
         self.state = :disconnected
         raise Exceptions::ConnectivityFailure.new(not_responding.message)
+      when 0
+        @stats["request failures"].update("#{type} - no result")
+        self.state = :disconnected
+        raise Exceptions::ConnectivityFailure.new(not_responding.message + " after #{attempts} attempts")
+      else
+        Log.error("Retrying #{type} request <#{request_uuid}> in #{interval} seconds " +
+                  "in response to routing failure (#{BalancedHttpClient.exception_text(not_responding)})")
+        wait(interval)
       end
       true
     end
 
+    # Determine time interval before next retry
+    #
+    # @param [Time] expires_at time for request
+    # @param [Integer] attempts so far
+    # @param [Integer] max_retries
+    #
+    # @return [Integer, NilClass] retry interval with 0 meaning no try and nil meaning retry disabled
+    def retry_interval(expires_at, attempts, max_retries = nil)
+      if @options[:retry_enabled]
+        if max_retries.nil? || attempts <= max_retries
+          interval = @options[:retry_intervals][attempts - 1] || @options[:retry_intervals][-1]
+          ((Time.now + interval) < expires_at) ? interval : 0
+        else
+          0
+        end
+      end
+    end
+
     # Wait the specified interval in non-blocking fashion if possible
     #
     # @param [Numeric] interval to wait