httpx 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5770fadc8d4604ccb0fb274c7fc157802315f68c749a83973ee5596fee8effb1
-  data.tar.gz: 4f24cca053093be31636405dcb740f5c6b2599197d9ebdfb90cbf127e86a5ffc
+  metadata.gz: ecaac1dac4ce953ba1585a418432d9bf635aea581a6d81acb32778ff947f486b
+  data.tar.gz: 32d9c5f65bd18ceeab524c6734081740ea2a3cbe526d3a6a71e079846f047644
 SHA512:
-  metadata.gz: 48fe64207a82e2b8db91b73cc6252ff48ea624e57dcbcbfa30e5f6986e1936e61bbdc83cece34c470dd6e318f41845a306a57e2b5c8710084444b6193786a1eb
-  data.tar.gz: 70b43fc624a8452187a730e39f55d9e92d16438babbd76aa2346c5b3dd0c2cad7f8e5e787f7c4ab1917c52ed750c7656f2f6c5e92a2ec0fce9cc0a76e5994d1e
+  metadata.gz: 204b0aea0e3bc09eccd31c9b93c79de866cff737801bca75096a61ea3ff45e62d44b2e9d39e26e3fb34d261dc518e17766080549f55b1cf0e16cfcd55565ddb4
+  data.tar.gz: ffc06e694b6afd9e29d48857c0251b75829d41637583f45243dd7b30931f0b9a4f8bf06d13421e94dc5f07bf5113ff0f9f2c888be6313c9506e6da05659392a1

data/doc/release_notes/0_13_0.md

@@ -0,0 +1,58 @@
+# 0.13.0
+
+## Features
+
+### Upgrade plugin
+
+A new plugin, `:upgrade`, is now available. This plugin allows one to "hook" on HTTP/1.1's protocol upgrade mechanism (see: https://developer.mozilla.org/en-US/docs/Web/HTTP/Protocol_upgrade_mechanism), which is the mechanism that browsers use to initiate websockets (there is an example of how to use `httpx` to start a websocket client connection [in the tests](https://gitlab.com/honeyryderchuck/httpx/-/blob/master/test/support/requests/plugins/upgrade.rb))
+
+You can read more about the `:upgrade` plugin in the [wiki](https://honeyryderchuck.gitlab.io/httpx/wiki/Connection-Upgrade).
+
+It's the basis of two plugins:
+
+#### `:h2c`
+
+This plugin was been rewritten on top of the `:upgrade` plugin, and handles upgrading a plaintext (non-"https") HTTP/1.1 connection, into an HTTP/2 connection.
+
+https://honeyryderchuck.gitlab.io/httpx/wiki/Connection-Upgrade#h2c
+
+#### `:upgrade/h2`
+
+This plugin handles when a server responds to a request with an `Upgrade: h2` header, does the following requests to the same origin via HTTP/2 prior knowledge (bypassing the necessity for ALPN negotiation, which is the whole point of the feature).
+
+https://honeyryderchuck.gitlab.io/httpx/wiki/Connection-Upgrade#h2
+
+### `:addresses` option
+
+The `:addresses` option is now available. You can use it to pass a list of IPs to connect to:
+
+```ruby
+# will not resolve example.com, and instead connect to one of the IPs passed.
+HTTPX.get("http://example.com", addresses: %w[172.5.3.1 172.5.3.2]))
+```
+
+You should also use it to connect to HTTP servers bound to a UNIX socket, in which case you'll have to provide a path:
+
+```ruby
+HTTPX.get("http://example.com", addresses: %w[/path/to/usocket]))
+```
+
+The `:transport_options` are therefore deprecated, and will be moved in a major version.
+
+## Improvements
+
+Some internal improvements that allow certainn plugins not to "leak" globally, such as the `:compression` plugin, which used to enable compression for all the `httpx` sessions from the same process. It doesn't anymore.
+
+Using exceptionless nonblocking connect calls in the supported rubies.
+
+Removed unneeded APIs around the Options object (`with_` methods, or the defined options list).
+
+## Bugfixes
+
+HTTP/1.1 persistent connections were closing after each request after the max requests was reached. It's fixed, and the new connection will also be persistent.
+
+When passing open IO objects for origins (the `:io` option), `httpx` was still trying to resolve the origin's domain. This not only didn't make sense, it broke if the domain is unresolvable. It has been fixed.
+
+Fixed usage of `:io` option when passed an "authority/io" hash.
+
+Fixing some issues around trying to connnect to the next available IPAddress when the previous one was unreachable or ETIMEDOUT.

data/lib/httpx/chainable.rb

@@ -17,12 +17,12 @@ module HTTPX
     # :nocov:
     def timeout(**args)
       warn ":#{__method__} is deprecated, use :with_timeout instead"
-      branch(default_options.with(timeout: args))
+      with(timeout: args)
     end
 
     def headers(headers)
       warn ":#{__method__} is deprecated, use :with_headers instead"
-      branch(default_options.with(headers: headers))
+      with(headers: headers)
     end
     # :nocov:
 

data/lib/httpx/connection.rb

@@ -71,6 +71,8 @@ module HTTPX
       @inflight = 0
       @keep_alive_timeout = options.timeout.keep_alive_timeout
       @keep_alive_timer = nil
+
+      self.addresses = options.addresses if options.addresses
     end
 
     # this is a semi-private method, to be used by the resolver
@@ -105,6 +107,8 @@ module HTTPX
 
       return false if exhausted?
 
+      return false unless connection.addresses
+
       !(@io.addresses & connection.addresses).empty? && @options == connection.options
     end
 
@@ -472,26 +476,15 @@ module HTTPX
           remove_instance_variable(:@total_timeout)
         end
 
-        @io.close if @io
-        @read_buffer.clear
-        if @keep_alive_timer
-          @keep_alive_timer.cancel
-          remove_instance_variable(:@keep_alive_timer)
-        end
-
-        remove_instance_variable(:@timeout) if defined?(@timeout)
+        purge_after_closed
       when :already_open
         nextstate = :open
         send_pending
       end
       @state = nextstate
-    rescue Errno::EHOSTUNREACH
-      # at this point, all addresses from the IO object have failed
-      reset
-      emit(:unreachable)
-      throw(:jump_tick)
     rescue Errno::ECONNREFUSED,
            Errno::EADDRNOTAVAIL,
+           Errno::EHOSTUNREACH,
            TLSError => e
       # connect errors, exit gracefully
       handle_error(e)
@@ -499,6 +492,17 @@ module HTTPX
       emit(:close)
     end
 
+    def purge_after_closed
+      @io.close if @io
+      @read_buffer.clear
+      if @keep_alive_timer
+        @keep_alive_timer.cancel
+        remove_instance_variable(:@keep_alive_timer)
+      end
+
+      remove_instance_variable(:@timeout) if defined?(@timeout)
+    end
+
     def handle_response
       @inflight -= 1
       return unless @inflight.zero?

data/lib/httpx/connection/http1.rb

@@ -10,7 +10,7 @@ module HTTPX
     MAX_REQUESTS = 100
     CRLF = "\r\n"
 
-    attr_reader :pending
+    attr_reader :pending, :requests
 
     def initialize(buffer, options)
       @options = Options.new(options)
@@ -69,7 +69,6 @@ module HTTPX
 
       return if @requests.include?(request)
 
-      request.once(:headers, &method(:set_protocol_headers))
       @requests << request
       @pipelining = true if @requests.size > 1
     end
@@ -236,6 +235,8 @@ module HTTPX
 
     def disable_pipelining
       return if @requests.empty?
+      # do not disable pipelining if already set to 1 request at a time
+      return if @max_concurrent_requests == 1
 
       @requests.each do |r|
         r.transition(:idle)
@@ -281,6 +282,7 @@ module HTTPX
       log(color: :yellow) { "<- HEADLINE: #{buffer.chomp.inspect}" }
       @buffer << buffer
       buffer.clear
+      set_protocol_headers(request)
       request.headers.each do |field, value|
         buffer << "#{capitalized(field)}: #{value}" << CRLF
         log(color: :yellow) { "<- HEADER: #{buffer.chomp}" }

data/lib/httpx/connection/http2.rb

@@ -91,7 +91,6 @@ module HTTPX
         @streams[request] = stream
         @max_requests -= 1
       end
-      request.once(:headers, &method(:set_protocol_headers))
       handle(request, stream)
       true
     rescue HTTP2Next::Error::StreamLimitExceeded
@@ -187,6 +186,7 @@ module HTTPX
     end
 
     def join_headers(stream, request)
+      set_protocol_headers(request)
       log(level: 1, color: :yellow) do
         request.headers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
       end

data/lib/httpx/io/ssl.rb

@@ -47,10 +47,6 @@ module HTTPX
 
     def connect
       super
-      if @keep_open
-        @state = :negotiated
-        return
-      end
       return if @state == :negotiated ||
                 @state != :connected
 
@@ -59,17 +55,22 @@ module HTTPX
         @io.hostname = @sni_hostname
         @io.sync_close = true
       end
-      @io.connect_nonblock
-      @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
-      transition(:negotiated)
-      @interests = :w
-    rescue ::IO::WaitReadable
-      @interests = :r
-    rescue ::IO::WaitWritable
-      @interests = :w
+      try_ssl_connect
     end
 
     if RUBY_VERSION < "2.3"
+      # :nocov:
+      def try_ssl_connect
+        @io.connect_nonblock
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        transition(:negotiated)
+        @interests = :w
+      rescue ::IO::WaitReadable
+        @interests = :r
+      rescue ::IO::WaitWritable
+        @interests = :w
+      end
+
       def read(_, buffer)
         super
       rescue ::IO::WaitWritable
@@ -82,7 +83,23 @@ module HTTPX
       rescue ::IO::WaitReadable
         0
       end
+      # :nocov:
     else
+      def try_ssl_connect
+        case @io.connect_nonblock(exception: false)
+        when :wait_readable
+          @interests = :r
+          return
+        when :wait_writable
+          @interests = :w
+          return
+        end
+        @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
+        transition(:negotiated)
+        @interests = :w
+      end
+
+      # :nocov:
       if OpenSSL::VERSION < "2.0.6"
         def read(size, buffer)
           @io.read_nonblock(size, buffer)
@@ -95,11 +112,7 @@ module HTTPX
           nil
         end
       end
-    end
-
-    def inspect
-      id = @io.closed? ? "closed" : @io.to_io.fileno
-      "#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
+      # :nocov:
     end
 
     private

data/lib/httpx/io/tcp.rb

@@ -7,6 +7,8 @@ module HTTPX
   class TCP
     include Loggable
 
+    using URIExtensions
+
     attr_reader :ip, :port, :addresses, :state, :interests
 
     alias_method :host, :ip
@@ -14,7 +16,6 @@ module HTTPX
     def initialize(origin, addresses, options)
       @state = :idle
       @hostname = origin.host
-      @addresses = addresses
       @options = Options.new(options)
       @fallback_protocol = @options.fallback_protocol
       @port = origin.port
@@ -26,17 +27,17 @@ module HTTPX
               else
                 @options.io
         end
+        raise Error, "Given IO objects do not match the request authority" unless @io
+
         _, _, _, @ip = @io.addr
         @addresses ||= [@ip]
         @ip_index = @addresses.size - 1
-        unless @io.nil?
-          @keep_open = true
-          @state = :connected
-        end
+        @keep_open = true
+        @state = :connected
       else
-        @ip_index = @addresses.size - 1
-        @ip = @addresses[@ip_index]
+        @addresses = addresses.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
       end
+      @ip_index = @addresses.size - 1
       @io ||= build_socket
     end
 
@@ -51,17 +52,11 @@ module HTTPX
     def connect
       return unless closed?
 
-      begin
-        if @io.closed?
-          transition(:idle)
-          @io = build_socket
-        end
-        @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
-      rescue Errno::EISCONN
+      if @io.closed?
+        transition(:idle)
+        @io = build_socket
       end
-      @interests = :w
-
-      transition(:connected)
+      try_connect
     rescue Errno::EHOSTUNREACH => e
       raise e if @ip_index <= 0
 
@@ -72,15 +67,25 @@ module HTTPX
 
       @ip_index -= 1
       retry
-    rescue Errno::EINPROGRESS,
-           Errno::EALREADY
-      @interests = :w
-    rescue ::IO::WaitReadable
-      @interests = :r
     end
 
     if RUBY_VERSION < "2.3"
       # :nocov:
+      def try_connect
+        @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
+      rescue ::IO::WaitWritable, Errno::EALREADY
+        @interests = :w
+      rescue ::IO::WaitReadable
+        @interests = :r
+      rescue Errno::EISCONN
+        transition(:connected)
+        @interests = :w
+      else
+        transition(:connected)
+        @interests = :w
+      end
+      private :try_connect
+
       def read(size, buffer)
         @io.read_nonblock(size, buffer)
         log { "READ: #{buffer.bytesize} bytes..." }
@@ -104,6 +109,20 @@ module HTTPX
       end
       # :nocov:
     else
+      def try_connect
+        case @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
+        when :wait_readable
+          @interests = :r
+          return
+        when :wait_writable
+          @interests = :w
+          return
+        end
+        transition(:connected)
+        @interests = :w
+      end
+      private :try_connect
+
       def read(size, buffer)
         ret = @io.read_nonblock(size, buffer, exception: false)
         if ret == :wait_readable
@@ -148,14 +167,14 @@ module HTTPX
 
     # :nocov:
     def inspect
-      id = @io.closed? ? "closed" : @io.fileno
-      "#<TCP(fd: #{id}): #{@ip}:#{@port} (state: #{@state})>"
+      "#<#{self.class}: #{@ip}:#{@port} (state: #{@state})>"
     end
     # :nocov:
 
     private
 
     def build_socket
+      @ip = @addresses[@ip_index]
       Socket.new(@ip.family, :STREAM, 0)
     end
 
@@ -178,9 +197,9 @@ module HTTPX
     def log_transition_state(nextstate)
       case nextstate
       when :connected
-        "Connected to #{@hostname} (#{@ip}) port #{@port} (##{@io.fileno})"
+        "Connected to #{host} (##{@io.fileno})"
       else
-        "#{@ip}:#{@port} #{@state} -> #{nextstate}"
+        "#{host} #{@state} -> #{nextstate}"
       end
     end
   end

data/lib/httpx/io/unix.rb

@@ -6,34 +6,43 @@ module HTTPX
   class UNIX < TCP
     extend Forwardable
 
-    def_delegator :@uri, :port, :scheme
+    using URIExtensions
 
-    def initialize(uri, addresses, options)
-      @uri = uri
+    attr_reader :path
+
+    alias_method :host, :path
+
+    def initialize(origin, addresses, options)
       @addresses = addresses
+      @hostname = origin.host
       @state = :idle
       @options = Options.new(options)
-      @path = @options.transport_options[:path]
       @fallback_protocol = @options.fallback_protocol
       if @options.io
         @io = case @options.io
               when Hash
-                @options.io[@path]
+                @options.io[origin.authority]
               else
                 @options.io
         end
-        unless @io.nil?
-          @keep_open = true
-          @state = :connected
+        raise Error, "Given IO objects do not match the request authority" unless @io
+
+        @path = @io.path
+        @keep_open = true
+        @state = :connected
+      else
+        if @options.transport_options
+          # :nocov:
+          warn ":#{__method__} is deprecated, use :addresses instead"
+          @path = @options.transport_options[:path]
+          # :nocov:
+        else
+          @path = addresses.first
         end
       end
       @io ||= build_socket
     end
 
-    def hostname
-      @uri.host
-    end
-
     def connect
       return unless closed?
 
@@ -51,6 +60,12 @@ module HTTPX
            ::IO::WaitReadable
     end
 
+    # :nocov:
+    def inspect
+      "#<#{self.class}(path: #{@path}): (state: #{@state})>"
+    end
+    # :nocov:
+
     private
 
     def build_socket