httpx 0.3.1 → 0.4.0

data/lib/httpx/plugins/retries.rb

@@ -2,9 +2,37 @@
 
 module HTTPX
   module Plugins
+    #
+    # This plugin adds support for retrying requests when certain errors happen.
+    #
     module Retries
       MAX_RETRIES = 3
+      # TODO: pass max_retries in a configure/load block
+
       IDEMPOTENT_METHODS = %i[get options head put delete].freeze
+      RETRYABLE_ERRORS = [IOError,
+                          EOFError,
+                          Errno::ECONNRESET,
+                          Errno::ECONNABORTED,
+                          Errno::EPIPE,
+                          (OpenSSL::SSL::SSLError if defined?(OpenSSL)),
+                          TimeoutError,
+                          Parser::Error,
+                          Errno::EINVAL,
+                          Errno::ETIMEDOUT].freeze
+
+      def self.extra_options(options)
+        Class.new(options.class) do
+          def_option(:max_retries) do |num|
+            num = Integer(num)
+            raise Error, ":max_retries must be positive" unless num.positive?
+
+            num
+          end
+
+          def_option(:retry_change_requests)
+        end.new(options)
+      end
 
       module InstanceMethods
         def max_retries(n)
@@ -13,18 +41,29 @@ module HTTPX
 
         private
 
-        def fetch_response(request)
+        def fetch_response(request, connections, options)
           response = super
           if response.is_a?(ErrorResponse) &&
              request.retries.positive? &&
-             IDEMPOTENT_METHODS.include?(request.verb)
+             __repeatable_request?(request, options) &&
+             __retryable_error?(response.error)
             request.retries -= 1
-            channel = find_channel(request)
-            channel.send(request)
+            log { "failed to get response, #{request.retries} tries to go..." }
+            request.transition(:idle)
+            connection = find_connection(request, connections, options)
+            connection.send(request)
             return
           end
           response
         end
+
+        def __repeatable_request?(request, options)
+          IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
+        end
+
+        def __retryable_error?(ex)
+          RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
+        end
       end
 
       module RequestMethods
@@ -35,17 +74,6 @@ module HTTPX
           @retries = @options.max_retries || MAX_RETRIES
         end
       end
-
-      module OptionsMethods
-        def self.included(klass)
-          super
-          klass.def_option(:max_retries) do |num|
-            num = Integer(num)
-            raise Error, ":max_retries must be positive" unless num.positive?
-            num
-          end
-        end
-      end
     end
     register_plugin :retries, Retries
   end

data/lib/httpx/pool.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require "httpx/selector"
+require "httpx/connection"
+require "httpx/resolver"
+
+module HTTPX
+  class Pool
+    def initialize
+      @resolvers = {}
+      @_resolver_monitors = {}
+      @selector = Selector.new
+      @connections = []
+      @connected_connections = 0
+    end
+
+    def empty?
+      @connections.empty?
+    end
+
+    def next_tick(timeout = nil)
+      catch(:jump_tick) do
+        tout = timeout.total_timeout if timeout
+
+        @selector.select(next_timeout || tout) do |monitor|
+          monitor.io.call
+          monitor.interests = monitor.io.interests
+        end
+      end
+    rescue StandardError => ex
+      @connections.each do |connection|
+        connection.emit(:error, ex)
+      end
+    end
+
+    def close(connections = @connections)
+      connections = connections.reject(&:inflight?)
+      connections.each(&:close)
+      next_tick until connections.none? { |c| @connections.include?(c) }
+      @resolvers.each_value do |resolver|
+        resolver.close unless resolver.closed?
+      end if @connections.empty?
+    end
+
+    def init_connection(connection, _options)
+      resolve_connection(connection)
+      connection.on(:open) do
+        @connected_connections += 1
+      end
+      connection.on(:unreachable) do
+        resolver = find_resolver_for(connection)
+        resolver.uncache(connection) if resolver
+        resolve_connection(connection)
+      end
+    end
+
+    # opens a connection to the IP reachable through +uri+.
+    # Many hostnames are reachable through the same IP, so we try to
+    # maximize pipelining by opening as few connections as possible.
+    #
+    def find_connection(uri, options)
+      @connections.find do |connection|
+        connection.match?(uri, options)
+      end
+    end
+
+    private
+
+    def resolve_connection(connection)
+      @connections << connection unless @connections.include?(connection)
+      resolver = find_resolver_for(connection)
+      resolver << connection
+      return if resolver.empty?
+
+      @_resolver_monitors[resolver] ||= @selector.register(resolver, :w)
+    end
+
+    def on_resolver_connection(connection)
+      found_connection = @connections.find do |ch|
+        ch != connection && ch.mergeable?(connection)
+      end
+      return register_connection(connection) unless found_connection
+
+      if found_connection.state == :open
+        coalesce_connections(found_connection, connection)
+      else
+        found_connection.once(:open) do
+          coalesce_connections(found_connection, connection)
+        end
+      end
+    end
+
+    def on_resolver_error(ch, error)
+      ch.emit(:error, error)
+      # must remove connection by hand, hasn't been started yet
+      unregister_connection(ch)
+    end
+
+    def on_resolver_close(resolver)
+      resolver_type = resolver.class
+      return unless @resolvers[resolver_type] == resolver
+
+      @resolvers.delete(resolver_type)
+
+      @selector.deregister(resolver)
+      monitor = @_resolver_monitors.delete(resolver)
+      monitor.close if monitor
+      resolver.close unless resolver.closed?
+    end
+
+    def register_connection(connection)
+      if connection.state == :open
+        # if open, an IO was passed upstream, therefore
+        # consider it connected already.
+        @connected_connections += 1
+        @selector.register(connection, :rw)
+      else
+        @selector.register(connection, :w)
+      end
+      connection.on(:close) do
+        unregister_connection(connection)
+      end
+      return if connection.state == :open
+    end
+
+    def unregister_connection(connection)
+      @connections.delete(connection)
+      @selector.deregister(connection)
+      @connected_connections -= 1
+    end
+
+    def coalesce_connections(ch1, ch2)
+      if ch1.coalescable?(ch2)
+        ch1.merge(ch2)
+        @connections.delete(ch2)
+      else
+        register_connection(ch2)
+      end
+    end
+
+    def next_timeout
+      @resolvers.values.reject(&:closed?).map(&:timeout).compact.min || @connections.map(&:timeout).compact.min
+    end
+
+    def find_resolver_for(connection)
+      connection_options = connection.options
+      resolver_type = connection_options.resolver_class
+      resolver_type = Resolver.registry(resolver_type) if resolver_type.is_a?(Symbol)
+
+      @resolvers[resolver_type] ||= begin
+        resolver = resolver_type.new(connection_options)
+        resolver.on(:resolve, &method(:on_resolver_connection))
+        resolver.on(:error, &method(:on_resolver_error))
+        resolver.on(:close) { on_resolver_close(resolver) }
+        resolver
+      end
+    end
+  end
+end

data/lib/httpx/registry.rb

@@ -58,8 +58,10 @@ module HTTPX
       def registry(tag = nil)
         @registry ||= {}
         return @registry if tag.nil?
+
         handler = @registry.fetch(tag)
         raise(Error, "#{tag} is not registered in #{self}") unless handler
+
         case handler
         when Symbol, String
           const_get(handler)

data/lib/httpx/request.rb

@@ -5,6 +5,7 @@ require "forwardable"
 module HTTPX
   class Request
     extend Forwardable
+    include Callbacks
     using URIExtensions
 
     METHODS = [
@@ -34,6 +35,8 @@ module HTTPX
 
     attr_reader :verb, :uri, :headers, :body, :state
 
+    attr_reader :options
+
     attr_accessor :response
 
     def_delegator :@body, :<<
@@ -84,6 +87,7 @@ module HTTPX
 
     def query
       return @query if defined?(@query)
+
       query = []
       if (q = @options.params)
         query << URI.encode_www_form(q)
@@ -94,6 +98,7 @@ module HTTPX
 
     def drain_body
       return nil if @body.nil?
+
       @drainer ||= @body.each
       chunk = @drainer.next
       chunk.dup
@@ -109,6 +114,7 @@ module HTTPX
       class << self
         def new(*, options)
           return options.body if options.body.is_a?(self)
+
           super
         end
       end
@@ -123,6 +129,7 @@ module HTTPX
           Transcoder.registry("json").encode(options.json)
         end
         return if @body.nil?
+
         @headers["content-type"] ||= @body.content_type
         @headers["content-length"] = @body.bytesize unless unbounded_body?
       end
@@ -130,6 +137,7 @@ module HTTPX
       def each(&block)
         return enum_for(__method__) unless block_given?
         return if @body.nil?
+
         body = stream(@body)
         if body.respond_to?(:read)
           ::IO.copy_stream(body, ProcIO.new(block))
@@ -143,11 +151,13 @@ module HTTPX
       def empty?
         return true if @body.nil?
         return false if chunked?
+
         bytesize.zero?
       end
 
       def bytesize
         return 0 if @body.nil?
+
         if @body.respond_to?(:bytesize)
           @body.bytesize
         elsif @body.respond_to?(:size)

data/lib/httpx/resolver.rb

@@ -52,6 +52,7 @@ module HTTPX
     # do not use directly!
     def lookup(hostname, ttl)
       return unless @lookups.key?(hostname)
+
       @lookups[hostname] = @lookups[hostname].select do |address|
         address["TTL"] > ttl
       end
@@ -92,7 +93,7 @@ module HTTPX
              Resolv::DNS::Resource::IN::AAAA
           addresses << {
             "name" => question.to_s,
-            "TTL"  => value.ttl,
+            "TTL" => value.ttl,
             "data" => value.address.to_s,
           }
         end

data/lib/httpx/resolver/https.rb

@@ -22,111 +22,111 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegator :@channels, :empty?
+    def_delegator :@connections, :empty?
 
-    def_delegators :@resolver_channel, :to_io, :call, :interests, :close
+    def_delegators :@resolver_connection, :to_io, :call, :interests, :close
 
-    def initialize(connection, options)
-      @connection = connection
+    def initialize(options)
       @options = Options.new(options)
       @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
       @_record_types = Hash.new { |types, host| types[host] = RECORD_TYPES.keys.dup }
       @queries = {}
       @requests = {}
-      @channels = []
+      @connections = []
       @uri = URI(@resolver_options.uri)
       @uri_addresses = nil
     end
 
-    def <<(channel)
+    def <<(connection)
       @uri_addresses ||= Resolv.getaddresses(@uri.host)
       if @uri_addresses.empty?
-        ex = ResolveError.new("Can't resolve #{channel.uri.host}")
+        ex = ResolveError.new("Can't resolve #{connection.origin.host}")
         ex.set_backtrace(caller)
-        emit(:error, channel, ex)
+        emit(:error, connection, ex)
       else
-        early_resolve(channel) || resolve(channel)
+        early_resolve(connection) || resolve(connection)
       end
     end
 
     def timeout
-      timeout = @options.timeout
-      timeout.timeout
+      @connections.map(&:timeout).min
     end
 
     def closed?
-      return true unless @resolver_channel
-      resolver_channel.closed?
+      return true unless @resolver_connection
+
+      resolver_connection.closed?
     end
 
     private
 
-    def resolver_channel
-      @resolver_channel ||= find_channel(@uri, @options)
+    def pool
+      Thread.current[:httpx_connection_pool] ||= Pool.new
     end
 
-    def resolve(channel = @channels.first, hostname = nil)
-      return if @building_channel
-      hostname = hostname || @queries.key(channel) || channel.uri.host
+    def resolver_connection
+      @resolver_connection ||= pool.find_connection(@uri, @options) || begin
+        @building_connection = true
+        connection = @options.connection_class.new("ssl", @uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
+        pool.init_connection(connection, @options)
+        emit_addresses(connection, @uri_addresses)
+        @building_connection = false
+        connection
+      end
+    end
+
+    def resolve(connection = @connections.first, hostname = nil)
+      return if @building_connection
+
+      hostname = hostname || @queries.key(connection) || connection.origin.host
       type = @_record_types[hostname].first
       log(label: "resolver: ") { "query #{type} for #{hostname}" }
       begin
         request = build_request(hostname, type)
-        @requests[request] = channel
-        resolver_channel.send(request)
-        @queries[hostname] = channel
-        @channels << channel
+        @requests[request] = connection
+        resolver_connection.send(request)
+        @queries[hostname] = connection
+        @connections << connection
       rescue Resolv::DNS::EncodeError, JSON::JSONError => e
-        emit_resolve_error(channel, hostname, e)
+        emit_resolve_error(connection, hostname, e)
       end
     end
 
-    def find_channel(_request, **options)
-      @connection.find_channel(@uri) || begin
-        @building_channel = true
-        channel = @connection.build_channel(@uri, **options)
-        emit_addresses(channel, @uri_addresses)
-        set_channel_callbacks(channel)
-        @building_channel = false
-        channel
-      end
-    end
-
-    def set_channel_callbacks(channel)
-      channel.on(:response, &method(:on_response))
-      channel.on(:promise, &method(:on_response))
-    end
-
     def on_response(request, response)
       response.raise_for_status
     rescue Error => ex
-      channel = @requests[request]
-      hostname = @queries.key(channel)
+      connection = @requests[request]
+      hostname = @queries.key(connection)
       error = ResolveError.new("Can't resolve #{hostname}: #{ex.message}")
       error.set_backtrace(ex.backtrace)
-      emit(:error, channel, error)
+      emit(:error, connection, error)
     else
       parse(response)
     ensure
       @requests.delete(request)
     end
 
+    def on_promise(_, stream)
+      log(level: 2, label: "#{stream.id}: ") { "refusing stream!" }
+      stream.refuse
+    end
+
     def parse(response)
-      answers = begin
-        decode_response_body(response)
+      begin
+        answers = decode_response_body(response)
       rescue Resolv::DNS::DecodeError, JSON::JSONError => e
-        host, channel = @queries.first
+        host, connection = @queries.first
         if @_record_types[host].empty?
-          emit_resolve_error(channel, host, e)
+          emit_resolve_error(connection, host, e)
           return
         end
       end
       if answers.empty?
-        host, channel = @queries.first
+        host, connection = @queries.first
         @_record_types[host].shift
         if @_record_types[host].empty?
           @_record_types.delete(host)
-          emit_resolve_error(channel, host)
+          emit_resolve_error(connection, host)
           return
         end
       else
@@ -136,9 +136,9 @@ module HTTPX
             if address.key?("alias")
               alias_address = answers[address["alias"]]
               if alias_address.nil?
-                channel = @queries[hostname]
+                connection = @queries[hostname]
                 @queries.delete(address["name"])
-                resolve(channel, address["alias"])
+                resolve(connection, address["alias"])
                 return # rubocop:disable Lint/NonLocalExitFromIterator
               else
                 alias_address
@@ -148,15 +148,18 @@ module HTTPX
             end
           end.compact
           next if addresses.empty?
+
           hostname = hostname[0..-2] if hostname.end_with?(".")
-          channel = @queries.delete(hostname)
-          next unless channel # probably a retried query for which there's an answer
-          @channels.delete(channel)
+          connection = @queries.delete(hostname)
+          next unless connection # probably a retried query for which there's an answer
+
+          @connections.delete(connection)
           Resolver.cached_lookup_set(hostname, addresses)
-          emit_addresses(channel, addresses.map { |addr| addr["data"] })
+          emit_addresses(connection, addresses.map { |addr| addr["data"] })
         end
       end
-      return if @channels.empty?
+      return if @connections.empty?
+
       resolve
     end
 
@@ -175,6 +178,8 @@ module HTTPX
         request.headers["content-type"] = "application/dns-message"
         request.headers["accept"] = "application/dns-message"
       end
+      request.on(:response, &method(:on_response).curry[request])
+      request.on(:promise, &method(:on_promise))
       request
     end
 
@@ -188,7 +193,8 @@ module HTTPX
       when "application/dns-udpwireformat",
            "application/dns-message"
         Resolver.decode_dns_answer(response.to_s)
-        # TODO: what about the rest?
+      else
+        raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
       end
     end
   end