httpx 0.16.1 → 0.18.2

data/lib/httpx/connection.rb

@@ -70,7 +70,7 @@ module HTTPX
 
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
-      @keep_alive_timer = nil
+      @total_timeout = @options.timeout[:total_timeout]
 
       self.addresses = @options.addresses if @options.addresses
     end
@@ -117,7 +117,8 @@ module HTTPX
     def coalescable?(connection)
       if @io.protocol == "h2" &&
          @origin.scheme == "https" &&
-         connection.origin.scheme == "https"
+         connection.origin.scheme == "https" &&
+         @io.can_verify_peer?
         @io.verify_hostname(connection.origin.host)
       else
         @origin == connection.origin
@@ -200,11 +201,9 @@ module HTTPX
     end
 
     def close
-      @parser.close if @parser
-      return unless @keep_alive_timer
+      transition(:active) if @state == :inactive
 
-      @keep_alive_timer.cancel
-      remove_instance_variable(:@keep_alive_timer)
+      @parser.close if @parser
     end
 
     def reset
@@ -216,26 +215,40 @@ module HTTPX
     def send(request)
       if @parser && !@write_buffer.full?
         request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
-        if @keep_alive_timer
+
+        if @response_received_at && @keep_alive_timeout &&
+           Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
           # when pushing a request into an existing connection, we have to check whether there
           # is the possibility that the connection might have extended the keep alive timeout.
           # for such cases, we want to ping for availability before deciding to shovel requests.
-          if @keep_alive_timer.fires_in.negative?
-            @pending << request
-            parser.ping
-            return
-          end
-
-          @keep_alive_timer.pause
+          @pending << request
+          parser.ping
+          transition(:active) if @state == :inactive
+          return
         end
-        @inflight += 1
-        parser.send(request)
+
+        send_request_to_parser(request)
       else
         @pending << request
       end
     end
 
     def timeout
+      if @total_timeout
+        return @total_timeout unless @connected_at
+
+        elapsed_time = @total_timeout - Utils.elapsed_time(@connected_at)
+
+        if elapsed_time.negative?
+          ex = TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds")
+          ex.set_backtrace(caller)
+          on_error(ex)
+          return
+        end
+
+        return elapsed_time
+      end
+
       return @timeout if defined?(@timeout)
 
       return @options.timeout[:connect_timeout] if @state == :idle
@@ -243,6 +256,14 @@ module HTTPX
       @options.timeout[:operation_timeout]
     end
 
+    def deactivate
+      transition(:inactive)
+    end
+
+    def open?
+      @state == :open || @state == :inactive
+    end
+
     private
 
     def connect
@@ -313,7 +334,7 @@ module HTTPX
 
             # exit #consume altogether if all outstanding requests have been dealt with
             return if @pending.size.zero? && @inflight.zero?
-          end unless (interests.nil? || interests == :w || @state == :closing) && !epiped
+          end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
 
           #
           # tight write loop.
@@ -360,19 +381,18 @@ module HTTPX
             break if interests == :r || @state == :closing || @state == :closed
 
             write_drained = false
-          end unless interests == :r
+          end unless (ints = interests) == :r
 
           send_pending if @state == :open
 
           # return if socket is drained
-          next unless (interests != :r || read_drained) &&
-                      (interests != :w || write_drained)
+          next unless (ints != :r || read_drained) && (ints != :w || write_drained)
 
           # gotta go back to the event loop. It happens when:
           #
           # * the socket is drained of bytes or it's not the interest of the conn to read;
           # * theres nothing more to write, or it's not in the interest of the conn to write;
-          log(level: 3) { "(#{interests}): WAITING FOR EVENTS..." }
+          log(level: 3) { "(#{ints}): WAITING FOR EVENTS..." }
           return
         end
       end
@@ -380,9 +400,7 @@ module HTTPX
 
     def send_pending
       while !@write_buffer.full? && (request = @pending.shift)
-        @inflight += 1
-        @keep_alive_timer.pause if @keep_alive_timer
-        parser.send(request)
+        send_request_to_parser(request)
       end
     end
 
@@ -390,6 +408,15 @@ module HTTPX
       @parser ||= build_parser
     end
 
+    def send_request_to_parser(request)
+      @inflight += 1
+      parser.send(request)
+
+      return unless @state == :inactive
+
+      transition(:active)
+    end
+
     def build_parser(protocol = @io.protocol)
       parser = registry(protocol).new(@write_buffer, @options)
       set_parser_callbacks(parser)
@@ -401,7 +428,8 @@ module HTTPX
         AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
           emit(:altsvc, alt_origin, origin, alt_params)
         end
-        handle_response
+        @response_received_at = Utils.now
+        @inflight -= 1
         request.emit(:response, response)
       end
       parser.on(:altsvc) do |alt_origin, origin, alt_params|
@@ -421,7 +449,7 @@ module HTTPX
       end
       parser.on(:close) do |force|
         transition(:closing)
-        if force
+        if force || @state == :idle
           transition(:closed)
           emit(:close)
         end
@@ -436,6 +464,7 @@ module HTTPX
           transition(:closing)
           transition(:closed)
           emit(:reset)
+
           @parser.reset if @parser
           transition(:idle)
           transition(:open)
@@ -467,15 +496,17 @@ module HTTPX
       when :open
         return if @state == :closed
 
-        total_timeout
-
         @io.connect
         return unless @io.connected?
 
+        @connected_at = Utils.now
+
         send_pending
 
         @timeout = @current_timeout = parser.timeout
         emit(:open)
+      when :inactive
+        return unless @state == :open
       when :closing
         return unless @state == :open
 
@@ -483,15 +514,15 @@ module HTTPX
         return unless @state == :closing
         return unless @write_buffer.empty?
 
-        if @total_timeout
-          @total_timeout.cancel
-          remove_instance_variable(:@total_timeout)
-        end
-
         purge_after_closed
       when :already_open
         nextstate = :open
         send_pending
+      when :active
+        return unless @state == :inactive
+
+        nextstate = :open
+        emit(:activate)
       end
       @state = nextstate
     rescue Errno::ECONNREFUSED,
@@ -507,44 +538,24 @@ module HTTPX
     def purge_after_closed
       @io.close if @io
       @read_buffer.clear
-      if @keep_alive_timer
-        @keep_alive_timer.cancel
-        remove_instance_variable(:@keep_alive_timer)
-      end
-
       remove_instance_variable(:@timeout) if defined?(@timeout)
     end
 
-    def handle_response
-      @inflight -= 1
-      return unless @inflight.zero?
-
-      if @keep_alive_timer
-        @keep_alive_timer.resume
-        @keep_alive_timer.reset
-      else
-        @keep_alive_timer = @timers.after(@keep_alive_timeout) do
-          unless @inflight.zero?
-            log { "(#{@origin}): keep alive timeout expired" }
-            parser.ping
-          end
-        end
-      end
-    end
-
     def on_error(error)
       if error.instance_of?(TimeoutError)
-        if @timeout
-          @timeout -= error.timeout
-          return unless @timeout <= 0
-        end
 
-        if @total_timeout && @total_timeout.fires_in.negative?
-          ex = TotalTimeoutError.new(@total_timeout.interval, "Timed out after #{@total_timeout.interval} seconds")
+        if @total_timeout && @connected_at &&
+           Utils.elapsed_time(@connected_at) > @total_timeout
+          ex = TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds")
           ex.set_backtrace(error.backtrace)
           error = ex
-        elsif connecting?
-          error = error.to_connection_error
+        else
+          if @timeout
+            @timeout -= error.timeout
+            return unless @timeout <= 0
+          end
+
+          error = error.to_connection_error if connecting?
         end
       end
       handle_error(error)
@@ -559,18 +570,5 @@ module HTTPX
         request.emit(:response, response)
       end
     end
-
-    def total_timeout
-      total = @options.timeout[:total_timeout]
-
-      return unless total
-
-      @total_timeout ||= @timers.after(total) do
-        ex = TotalTimeoutError.new(total, "Timed out after #{total} seconds")
-        ex.set_backtrace(caller)
-        on_error(ex)
-        @parser.close if @parser
-      end
-    end
   end
 end

data/lib/httpx/domain_name.rb

@@ -123,7 +123,7 @@ module HTTPX
 
       # RFC 6265 #4.1.1
       # Domain-value must be a subdomain.
-      @domain && self <= domain && domain <= @domain ? true : false
+      @domain && self <= domain && domain <= @domain
     end
 
     # def ==(other)

data/lib/httpx/extensions.rb

@@ -54,6 +54,51 @@ module HTTPX
     Numeric.__send__(:include, NegMethods)
   end
 
+  module HashExtensions
+    refine Hash do
+      def compact
+        h = {}
+        each do |key, value|
+          h[key] = value unless value == nil
+        end
+        h
+      end unless Hash.method_defined?(:compact)
+    end
+  end
+
+  module ArrayExtensions
+    refine Array do
+
+      def filter_map
+        return to_enum(:filter_map) unless block_given?
+
+        each_with_object([]) do |item, res|
+          processed = yield(item)
+          res << processed if processed
+        end
+      end unless Array.method_defined?(:filter_map)
+
+      def sum(accumulator = 0, &block)
+        values = block_given? ? map(&block) : self
+        values.inject(accumulator, :+)
+      end unless Array.method_defined?(:sum)
+    end
+  end
+
+  module IOExtensions
+    refine IO do
+      # provides a fallback for rubies where IO#wait isn't implemented,
+      # but IO#wait_readable and IO#wait_writable are.
+      def wait(timeout = nil, _mode = :read_write)
+        r, w = IO.select([self], [self], nil, timeout)
+
+        return unless r || w
+
+        self
+      end unless IO.method_defined?(:wait) && IO.instance_method(:wait).arity == 2
+    end
+  end
+
   module RegexpExtensions
     # If you wonder why this is there: the oauth feature uses a refinement to enhance the
     # Regexp class locally with #match? , but this is never tested, because ActiveSupport
@@ -77,13 +122,14 @@ module HTTPX
       end
 
       def authority
-        port_string = port == default_port ? nil : ":#{port}"
-        "#{host}#{port_string}"
-      end
+        return host if port == default_port
+
+        "#{host}:#{port}"
+      end unless URI::HTTP.method_defined?(:authority)
 
       def origin
         "#{scheme}://#{authority}"
-      end
+      end unless URI::HTTP.method_defined?(:origin)
 
       def altsvc_match?(uri)
         uri = URI.parse(uri)

data/lib/httpx/headers.rb

@@ -57,7 +57,7 @@ module HTTPX
     def merge(other)
       headers = dup
       other.each do |field, value|
-        headers[field] = value
+        headers[downcased(field)] = value
       end
       headers
     end

data/lib/httpx/io/ssl.rb

@@ -27,6 +27,10 @@ module HTTPX
       super
     end
 
+    def can_verify_peer?
+      @ctx.verify_mode == OpenSSL::SSL::VERIFY_PEER
+    end
+
     def verify_hostname(host)
       return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
       return false if !@io.respond_to?(:peer_cert) || @io.peer_cert.nil?
@@ -134,7 +138,7 @@ module HTTPX
       server_cert = @io.peer_cert
 
       "#{super}\n\n" \
-      "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
+        "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
         "ALPN, server accepted to use #{protocol}\n" \
         "Server certificate:\n" \
         " subject: #{server_cert.subject}\n" \

data/lib/httpx/io/tls.rb

@@ -194,15 +194,15 @@ module HTTPX
       server_cert = @peer_cert
 
       "#{super}\n\n" \
-        "SSL connection using #{@ctx.ssl_version} / #{Array(@ctx.cipher).first}\n" \
-        "ALPN, server accepted to use #{protocol}\n" +
+      "SSL connection using #{@ctx.ssl_version} / #{Array(@ctx.cipher).first}\n" \
+      "ALPN, server accepted to use #{protocol}\n" +
         (if server_cert
            "Server certificate:\n" \
-                 " subject: #{server_cert.subject}\n" \
-                 " start date: #{server_cert.not_before}\n" \
-                 " expire date: #{server_cert.not_after}\n" \
-                 " issuer: #{server_cert.issuer}\n" \
-                 " SSL certificate verify ok."
+             " subject: #{server_cert.subject}\n" \
+             " start date: #{server_cert.not_before}\n" \
+             " expire date: #{server_cert.not_after}\n" \
+             " issuer: #{server_cert.issuer}\n" \
+             " SSL certificate verify ok."
          else
            "SSL certificate verify failed."
          end

data/lib/httpx/loggable.rb

@@ -24,15 +24,13 @@ module HTTPX
       debug_stream << message
     end
 
-    if !Exception.instance_methods.include?(:full_message)
+    if Exception.instance_methods.include?(:full_message)
 
       def log_exception(ex, level: @options.debug_level, color: nil)
         return unless @options.debug
         return unless @options.debug_level >= level
 
-        message = +"#{ex.message} (#{ex.class})"
-        message << "\n" << ex.backtrace.join("\n") unless ex.backtrace.nil?
-        log(level: level, color: color) { message }
+        log(level: level, color: color) { ex.full_message }
       end
 
     else
@@ -41,7 +39,9 @@ module HTTPX
         return unless @options.debug
         return unless @options.debug_level >= level
 
-        log(level: level, color: color) { ex.full_message }
+        message = +"#{ex.message} (#{ex.class})"
+        message << "\n" << ex.backtrace.join("\n") unless ex.backtrace.nil?
+        log(level: level, color: color) { message }
       end
 
     end

data/lib/httpx/options.rb

@@ -39,6 +39,28 @@ module HTTPX
       :resolver_options => { cache: true },
     }.freeze
 
+    begin
+      module HashExtensions
+        refine Hash do
+          def >=(other)
+            Hash[other] <= self
+          end
+
+          def <=(other)
+            other = Hash[other]
+            return false unless size <= other.size
+
+            each do |k, v|
+              v2 = other.fetch(k) { return false }
+              return false unless v2 == v
+            end
+            true
+          end
+        end
+      end
+      using HashExtensions
+    end unless Hash.method_defined?(:>=)
+
     class << self
       def new(options = {})
         # let enhanced options go through
@@ -59,9 +81,9 @@ module HTTPX
       end
 
       def def_option(optname, *args, &block)
-        if args.size.zero? && !block_given?
+        if args.size.zero? && !block
           class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-            def option_#{optname}(v); v; end
+            def option_#{optname}(v); v; end # def option_smth(v); v; end
           OUT
           return
         end
@@ -71,15 +93,15 @@ module HTTPX
 
       def deprecated_def_option(optname, layout = nil, &interpreter)
         warn "DEPRECATION WARNING: using `def_option(#{optname})` for setting options is deprecated. " \
-          "Define module OptionsMethods and `def option_#{optname}(val)` instead."
+             "Define module OptionsMethods and `def option_#{optname}(val)` instead."
 
         if layout
           class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-            def option_#{optname}(value)
-              #{layout}
-            end
+            def option_#{optname}(value)  # def option_origin(v)
+              #{layout}                   #   URI(v)
+            end                           # end
           OUT
-        elsif block_given?
+        elsif interpreter
           define_method(:"option_#{optname}") do |value|
             instance_exec(value, &interpreter)
           end
@@ -89,7 +111,7 @@ module HTTPX
 
     def initialize(options = {})
       defaults = DEFAULT_OPTIONS.merge(options)
-      defaults.each do |(k, v)|
+      defaults.each do |k, v|
         next if v.nil?
 
         begin
@@ -163,6 +185,7 @@ module HTTPX
     end
 
     REQUEST_IVARS = %i[@params @form @json @body].freeze
+    private_constant :REQUEST_IVARS
 
     def ==(other)
       ivars = instance_variables | other.instance_variables
@@ -180,14 +203,14 @@ module HTTPX
     end
 
     def merge(other)
-      raise ArgumentError, "#{other.inspect} is not a valid set of options" unless other.respond_to?(:to_hash)
+      raise ArgumentError, "#{other} is not a valid set of options" unless other.respond_to?(:to_hash)
 
       h2 = other.to_hash
       return self if h2.empty?
 
       h1 = to_hash
 
-      return self if h1 == h2
+      return self if h1 >= h2
 
       merged = h1.merge(h2) do |_k, v1, v2|
         if v1.respond_to?(:merge) && v2.respond_to?(:merge)
@@ -201,10 +224,9 @@ module HTTPX
     end
 
     def to_hash
-      hash_pairs = instance_variables.map do |ivar|
-        [ivar[1..-1].to_sym, instance_variable_get(ivar)]
+      instance_variables.each_with_object({}) do |ivar, hs|
+        hs[ivar[1..-1].to_sym] = instance_variable_get(ivar)
       end
-      Hash[hash_pairs]
     end
 
     if RUBY_VERSION > "2.4.0"

data/lib/httpx/parser/http1.rb

@@ -60,7 +60,7 @@ module HTTPX
         (m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
           raise(Error, "wrong head line format")
         version, code, _ = m.captures
-        raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)
+        raise(Error, "unsupported HTTP version (HTTP/#{version})") unless version && VERSIONS.include?(version)
 
         @http_version = version.split(".").map(&:to_i)
         @status_code = code.to_i
@@ -72,9 +72,14 @@ module HTTPX
 
       def parse_headers
         headers = @headers
-        while (idx = @buffer.index("\n"))
-          line = @buffer.byteslice(0..idx).sub(/\s+\z/, "")
-          @buffer = @buffer.byteslice((idx + 1)..-1)
+        buffer = @buffer
+
+        while (idx = buffer.index("\n"))
+          line = buffer.byteslice(0..idx)
+          raise Error, "wrong header format" if line.start_with?("\s", "\t")
+
+          line.lstrip!
+          buffer = @buffer = buffer.byteslice((idx + 1)..-1)
           if line.empty?
             case @state
             when :headers
@@ -97,9 +102,8 @@ module HTTPX
           raise Error, "wrong header format" unless separator_index
 
           key = line.byteslice(0..(separator_index - 1))
-          raise Error, "wrong header format" if key.start_with?("\s", "\t")
 
-          key.strip!
+          key.rstrip! # was lstripped previously!
           value = line.byteslice((separator_index + 1)..-1)
           value.strip!
           raise Error, "wrong header format" if value.nil?

data/lib/httpx/plugins/aws_sdk_authentication.rb

@@ -8,6 +8,23 @@ module HTTPX
     # It requires the "aws-sdk-core" gem.
     #
     module AwsSdkAuthentication
+      # Mock configuration, to be used only when resolving credentials
+      class Configuration
+        attr_reader :profile
+
+        def initialize(profile)
+          @profile = profile
+        end
+
+        def respond_to_missing?(*)
+          true
+        end
+
+        def method_missing(*)
+          nil
+        end
+      end
+
       #
       # encapsulates access to an AWS SDK credentials store.
       #
@@ -30,23 +47,8 @@ module HTTPX
       end
 
       class << self
-        attr_reader :credentials, :region
-
         def load_dependencies(_klass)
           require "aws-sdk-core"
-
-          client = Class.new(Seahorse::Client::Base) do
-            @identifier = :httpx
-            set_api(Aws::S3::ClientApi::API)
-            add_plugin(Aws::Plugins::CredentialsConfiguration)
-            add_plugin(Aws::Plugins::RegionalEndpoint)
-            class << self
-              attr_reader :identifier
-            end
-          end.new
-
-          @credentials = Credentials.new(client.config[:credentials])
-          @region = client.config[:region]
         end
 
         def configure(klass)
@@ -56,6 +58,26 @@ module HTTPX
         def extra_options(options)
           options.merge(max_concurrent_requests: 1)
         end
+
+        def credentials(profile)
+          mock_configuration = Configuration.new(profile)
+          Credentials.new(Aws::CredentialProviderChain.new(mock_configuration).resolve)
+        end
+
+        def region(profile)
+          # https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb#L62
+          keys = %w[AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION]
+          env_region = ENV.values_at(*keys).compact.first
+          env_region = nil if env_region == ""
+          cfg_region = Aws.shared_config.region(profile: profile)
+          env_region || cfg_region
+        end
+      end
+
+      module OptionsMethods
+        def option_aws_profile(value)
+          String(value)
+        end
       end
 
       module InstanceMethods
@@ -64,9 +86,11 @@ module HTTPX
         # aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
         # aws_authentication()
         #
-        def aws_sdk_authentication(**options)
-          credentials = AwsSdkAuthentication.credentials
-          region = AwsSdkAuthentication.region
+        def aws_sdk_authentication(
+          credentials: AwsSdkAuthentication.credentials(@options.aws_profile),
+          region: AwsSdkAuthentication.region(@options.aws_profile),
+          **options
+        )
 
           aws_sigv4_authentication(
             credentials: credentials,