right_support 2.10.1 → 2.11.2

data/lib/right_support/net/request_balancer.rb

@@ -20,6 +20,8 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+require 'thread'
+
 module RightSupport::Net
   # Raised to indicate the (uncommon) error condition where a RequestBalancer rotated
   # through EVERY URL in a list without getting a non-nil, non-timeout response.
@@ -110,32 +112,55 @@ module RightSupport::Net
       end
     end
 
+    # no-op health-check
     DEFAULT_HEALTH_CHECK_PROC = Proc.new do |endpoint|
       true
     end
 
+    # debug mode
+    DEFAULT_DEBUG_MODE = ::ENV['DEBUG_MODE'] == 'true'
+
+    # default options
     DEFAULT_OPTIONS = {
-        :policy       => nil,
-        :retry        => DEFAULT_RETRY_PROC,
-        :fatal        => DEFAULT_FATAL_PROC,
-        :on_exception => nil,
-        :health_check => DEFAULT_HEALTH_CHECK_PROC
+      :policy       => nil,
+      :retry        => DEFAULT_RETRY_PROC,
+      :fatal        => DEFAULT_FATAL_PROC,
+      :on_exception => nil,
+      :health_check => DEFAULT_HEALTH_CHECK_PROC,
+      :resolve      => nil,    # not resolving DNS to IP(s) by default; rely on consul, etc.
+      :thread_safe  => false,  # not thread-safe by default,
+      :debug_mode   => nil     # infer from DEBUG_MODE
     }
 
     attr_reader :endpoints
 
     # Return the actual, potentially DNS-resolved endpoints that are used for requests.
-    # If the balancer was constructed with :resolve=>false, return self.endpoints.
+    # If the balancer was constructed with :resolve=>nil, return self.endpoints.
     #
     # @return [Array] collection of endpoints
     def resolved_endpoints
-      (@ips.nil? || @ips.empty?) ? @endpoints : @ips 
+      @synchronize.call do
+        (@ips.nil? || @ips.empty?) ? @endpoints : @ips
+      end
     end
 
     def self.request(endpoints, options={}, &block)
       new(endpoints, options).request(&block)
     end
 
+    # encapsulates exponential backoff/retry logic in a callback for use as the
+    # :retry option to request balancer.
+    def self.backoff_retry_callback(max_attempts)
+      lambda do |_, n|
+        if n < max_attempts
+          sleep 2 ** n
+          true
+        else
+          false
+        end
+      end
+    end
+
     # Constructor. Accepts a sequence of request endpoints which it shuffles randomly at
     # creation time; however, the ordering of the endpoints does not change thereafter
     # and the sequence is tried from the beginning for every request.
@@ -147,24 +172,45 @@ module RightSupport::Net
     # :resolve option allows the balancer to treat each backing server as a distinct
     # endpoint with its own health state, etc.
     #
-    # === Parameters
-    # endpoints(Array):: a set of network endpoints (e.g. HTTP URLs) to be load-balanced
-    #
-    # === Options
-    # retry:: a Class, array of Class or decision Proc to determine whether to keep retrying; default is to try all endpoints
-    # fatal:: a Class, array of Class, or decision Proc to determine whether an exception is fatal and should not be retried
-    # resolve(Integer):: how often to re-resolve DNS hostnames of endpoints; default is nil (never resolve)
-    # on_exception(Proc):: notification hook that accepts three arguments: whether the exception is fatal, the exception itself,
-    #   and the endpoint for which the exception happened
-    # health_check(Proc):: callback that allows balancer to check an endpoint health; should raise an exception if the endpoint
-    #   is not healthy
-    # on_health_change(Proc):: callback that is made when the overall health of the endpoints transition to a different level;
-    #   its single argument contains the new minimum health level
-    #
+    # @param [String|Array] endpoints (e.g. HTTP URLs) for balancing
+    # @param [Hash] options
+    # @option options [Integer|Proc] :retry callback to determine whether to
+    #  keep retrying; default is to try each endpoint at most once. can also be
+    #  passed as an integer which provides a set number of attempts with no
+    #  backoff. for retry with backoff use the backoff_retry_callback method.
+    # @option options [Integer] :resolve as a timeout in seconds to re-resolve
+    #  DNS hostnames of endpoints to IP addresses; default is nil (never).
+    # @option options [TrueClass|FalseClass] :thread_safe as true to guard the
+    #  balancer state with a mutex, false to be free-threaded (default). Ruby is
+    #  generally thread-safe because real concurrency does not exist and/or apps
+    #  consistently use the Rainbows gem to ensure one process per API handler.
+    # @option options [TrueClass|FalseClass] :debug_mode as true to log
+    #  additional error information as failures occur, false to only log error
+    #  summary after all retries fail or nil to infer from DEBUG_MODE (default).
+    # @option options [Proc] :fatal callback to determine whether an exception
+    #  is fatal and should not be retried.
+    # @option options [Proc] :on_exception notification hook that accepts three
+    #  arguments: whether the exception is fatal, the exception itself, and the
+    #  endpoint for which the exception happened
+    # @option options [Proc] :health_check callback that allows balancer to
+    #  check an endpoint health; should raise an exception if the endpoint is
+    #  not healthy
+    # @option options [Proc] :on_health_change callback that is made when the
+    #  overall health of the endpoints transition to a different level; its
+    #  single argument contains the new minimum health level
     def initialize(endpoints, options={})
       @options = DEFAULT_OPTIONS.merge(options)
 
-      unless endpoints && !endpoints.empty?
+      # provide thread-safety only when specified.
+      if @options[:thread_safe]
+        @mutex = ::Mutex.new
+        @synchronize = @mutex.method(:synchronize)
+      else
+        @synchronize = self.method(:free_threaded)
+      end
+
+      endpoints = Array(endpoints)
+      if endpoints.empty?
         raise ArgumentError, "Must specify at least one endpoint"
       end
 
@@ -172,11 +218,32 @@ module RightSupport::Net
       @policy = @options[:policy]
       @policy = @policy.new(options) if @policy.is_a?(Class)
 
+      if (@debug_mode = @options.delete(:debug_mode)).nil?
+        @debug_mode = DEFAULT_DEBUG_MODE
+      end
+
+      # convert retry counter to a simple retry callback, if necessary.
+      @retry = @options.delete(:retry) || DEFAULT_RETRY_PROC
+      unless @retry.kind_of?(Proc)
+        # ensure that the count is captured by callback for safety.
+        @retry = Integer(@retry)
+        retry_proc = lambda do |max_attempts|
+          lambda do |ep, n|
+            n < max_attempts
+          end
+        end.call(@retry)
+        @retry = retry_proc  # and now the type always Proc
+      end
+
       unless test_policy_duck_type(@policy)
         raise ArgumentError, ":policy must be a class/object that responds to :next, :good and :bad"
       end
 
-      unless test_callable_arity(options[:retry], 2)
+      # note @retry is now always defined as a callback. the legacy code always
+      # had a default retry but it could have been defined later during actual
+      # execution instead of being concretely defined on initialization. now it
+      # is always defined on initialization.
+      unless test_callable_arity(@retry, 2, false)
         raise ArgumentError, ":retry callback must accept two parameters"
       end
 
@@ -209,13 +276,17 @@ module RightSupport::Net
 
     # Un-resolve an IP address.
     #
-    # === Parameters
-    # endpoint:: a network endpoint (e.g. HTTP URL) to be un-resolved
+    # @param [String] endpoint (e.g. HTTP URL) to be un-resolved
     #
-    # === Return
-    # Return the first hostname that resolved to the IP (there should only ever be one)
+    # @return [String] the first hostname that resolved to the IP (there should be at most one) or nil
     def lookup_hostname(endpoint)
-      @resolved_hostnames.select{ |k,v| v.addresses.include?(endpoint) }.shift[0]
+      result = nil
+      @synchronize.call do
+        if resolved_hostname = @resolved_hostnames && @resolved_hostnames.select{ |k, v| v.addresses.include?(endpoint) }
+          result = resolved_hostname.shift[0]
+        end
+      end
+      result
     end
 
     # Perform a request.
@@ -233,8 +304,9 @@ module RightSupport::Net
     # Return the first non-nil value provided by the block.
     def request
       raise ArgumentError, "Must call this method with a block" unless block_given?
-
-      resolve if need_resolve?
+      @synchronize.call do
+        resolve if need_resolve?
+      end
 
       exceptions = {}
       result     = nil
@@ -243,12 +315,27 @@ module RightSupport::Net
 
       loop do
         if n > 0
-          do_retry = @options[:retry] || DEFAULT_RETRY_PROC
-          do_retry = do_retry.call((@ips.nil? || @ips.empty?) ? @endpoints : @ips, n) if do_retry.respond_to?(:call)
-          break if (do_retry.is_a?(Integer) && n >= do_retry) || [nil, false].include?(do_retry)
+          retry_result = nil
+          @synchronize.call do
+            retry_result = @retry.call((@ips.nil? || @ips.empty?) ? @endpoints : @ips, n)
+          end
+
+          # FIX: this integer result logic is odd but is left for legacy support
+          # reasons. technically any retry proc could return integer and invoke
+          # this odd side-effect, which was only intended to support :retry as
+          # a literal integer. retry proc implementations should now only return
+          # boolean to avoid this weirdness. this logic should be removed in v3.
+          if retry_result.is_a?(Integer) && n >= retry_result
+            retry_result = false
+          end
+          break unless retry_result
         end
 
-        endpoint, need_health_check  = @policy.next
+        endpoint = nil
+        need_health_check = false
+        @synchronize.call do
+          endpoint, need_health_check = @policy.next
+        end
         break unless endpoint
 
         n += 1
@@ -257,41 +344,57 @@ module RightSupport::Net
         # Perform health check if necessary. Note that we guard this with a rescue, because the
         # health check may raise an exception and we want to log the exception info if this happens.
         if need_health_check
-          begin
-            unless @policy.health_check(endpoint)
-              logger.error "RequestBalancer: health check failed to #{endpoint} because of non-true return value"
-              next
+          hc_result = false
+          hc_exception = nil
+          @synchronize.call do
+            begin
+              # note that health-check can update the policy's good/bad state
+              # for endpoints.
+              hc_result = @policy.health_check(endpoint)
+            rescue Exception => e
+              hc_exception = e
             end
-          rescue Exception => e
-            logger.error "RequestBalancer: health check failed to #{endpoint} because of #{e.class.name}: #{e.message}"
-            if fatal_exception?(e)
+          end
+          if hc_result
+            logger.info "RequestBalancer: health check succeeded to #{endpoint}"
+          elsif hc_exception
+            logger.error "RequestBalancer: health check failed to #{endpoint} because of #{hc_exception.class.name}: #{hc_exception.message}"
+            if fatal_exception?(hc_exception)
               # Fatal exceptions should still raise, even if only during a health check
-              raise
+              raise hc_exception
             else
               # Nonfatal exceptions: keep on truckin'
+              exceptions[endpoint] ||= []
+              exceptions[endpoint] << hc_exception
+              debug_exception(hc_exception) if @debug_mode
               next
             end
+          else
+            logger.error "RequestBalancer: health check failed to #{endpoint} because of non-true return value"
+            next
           end
-
-          logger.info "RequestBalancer: health check succeeded to #{endpoint}"
         end
 
         begin
-          result   = yield(endpoint)
-          @policy.good(endpoint, t0, Time.now)
+          result = yield(endpoint)
+          @synchronize.call do
+            @policy.good(endpoint, t0, Time.now)
+          end
           complete = true
           break
         rescue Exception => e
           if to_raise = handle_exception(endpoint, e, t0)
             raise(to_raise)
           else
-            @policy.bad(endpoint, t0, Time.now)
+            @synchronize.call do
+              @policy.bad(endpoint, t0, Time.now)
+            end
             exceptions[endpoint] ||= []
             exceptions[endpoint] << e
+            debug_exception(e) if @debug_mode
           end
         end
-
-      end
+      end # loop
 
       return result if complete
 
@@ -302,15 +405,13 @@ module RightSupport::Net
         summary = []
         list.each { |e| summary << e.class }
         health = stats[endpoint] if stats[endpoint] != 'n/a'
-        if @resolved_hostnames
-          hostname = lookup_hostname(endpoint)
+        if hostname = lookup_hostname(endpoint)
           msg << "'#{hostname}' (#{endpoint}#{", "+health if health}) => [#{summary.uniq.join(', ')}]"
         else
           msg << "'#{endpoint}' #{"("+health+")" if health} => [#{summary.uniq.join(', ')}]"
         end
       end
-      message = "Request failed after #{n} tries to #{exceptions.keys.size} endpoints: (#{msg.join(', ')})"
-
+      message = "Request failed after #{n} tries to #{exceptions.size} endpoints: (#{msg.join(', ')})"
       logger.error "RequestBalancer: #{message}"
       raise NoResult.new(message, exceptions)
     end
@@ -332,10 +433,15 @@ module RightSupport::Net
     #
     # {2 => "n/a", 1 => "n/a", 3 => "n/a"}
     def get_stats
-      stats = {}
-      @endpoints.each { |endpoint| stats[endpoint] = 'n/a' }
-      stats = @policy.get_stats if @policy.respond_to?(:get_stats)
-      stats
+      result = nil
+      if @policy.respond_to?(:get_stats)
+        @synchronize.call do
+          result = @policy.get_stats
+        end
+      else
+        result = @endpoints.inject({}) { |h, endpoint| h[endpoint] = 'n/a'; h }
+      end
+      result
     end
 
     protected
@@ -345,9 +451,13 @@ module RightSupport::Net
     def handle_exception(endpoint, e, t0)
       fatal    = fatal_exception?(e)
       duration = sprintf('%.4f', Time.now - t0)
-      ept      = @resolved_hostnames ? "#{lookup_hostname(endpoint)}(#{endpoint})" : "#{endpoint}"
-      msg      = "RequestBalancer: rescued #{fatal ? 'fatal' : 'retryable'} #{e.class.name} " + 
-                 "during request to #{ept}: #{e.message} after #{duration} seconds"
+      if hostname = lookup_hostname(endpoint)
+        ept = "#{hostname}(#{endpoint})"
+      else
+        ept = endpoint
+      end
+      msg = "RequestBalancer: rescued #{fatal ? 'fatal' : 'retryable'} #{e.class.name} " +
+            "during request to #{ept}: #{e.message} after #{duration} seconds"
       logger.error msg
       @options[:on_exception].call(fatal, e, endpoint) if @options[:on_exception]
 
@@ -385,6 +495,7 @@ module RightSupport::Net
       @ips = resolved_endpoints
       @policy.set_endpoints(@ips)
       @resolved_at = Time.now.to_i
+      true
     end
 
     def need_resolve?
@@ -402,6 +513,19 @@ module RightSupport::Net
       return true if optional && !callable.respond_to?(:call)
       return callable.respond_to?(:arity) && (callable.arity == arity)
     end
+
+    # free-threaded invocation of the provided callback.
+    def free_threaded
+      yield
+    end
+
+    # logs exception with backtrace truncation for debug purposes only.
+    def debug_exception(e)
+      if (lines = e.backtrace || []).size > 7
+        lines = lines[0, 7] << '...'
+      end
+      logger.debug((["#{e.class}: #{e.message}"] + lines).join("\n"))
+    end
   end # RequestBalancer
 
 end # RightScale