httpx 1.1.5 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d95b9f470645015a3308ade4ab2349375eddd0598f6919fbf063d4feba61926c
-  data.tar.gz: 95518fc5601eb0ba9a22e13814f9f0a339addca71e94c33122ed211780049ca4
+  metadata.gz: f3df88a59256b85aacf9d98578c965626b3bfa4611695ae21a707252df71cb0a
+  data.tar.gz: f1616869724c217272d0dc165130548edc8ca35eadd97ab42d5a17012a0019d6
 SHA512:
-  metadata.gz: 729bfc7fc5888f6872ecf9dfdcaefa6df68ba96b705cf1ccd1e8b9b866d25bf2c3081577b3c1828b99556009fc686c0705eb30dcb6fd5f2a3fb951128e0c621e
-  data.tar.gz: 1173ae3cd242caf251c099e14db090efc3ac3524d8e3f63e04c331f2938a6d747c9874999d79dbc4790894fc8c447c900740235a62cb0376f2b124dad3674acc
+  metadata.gz: b4ca8efc0a1ffe3dec2d2d6ee345bcb58bc2eec9e4c5b4fe951a889041be8eeaeb4b005d9771f0e712f161dad69f8de25d07d520c3e0cd17ae1574f53fadace1
+  data.tar.gz: 509c48d74aafb520e3142c5f9b2356e2f36bf9d649eb723ca07a2863077057ab12ae2e17dbf61f13149eac4f80201252dad9b46cb904f5664b8910b62a868c1d

data/README.md

@@ -61,7 +61,7 @@ puts body #=> #<HTTPX::Response ...
 You can also send as many requests as you want simultaneously:
 
 ```ruby
-page1, page2, page3 =`HTTPX.get("https://news.ycombinator.com/news", "https://news.ycombinator.com/news?p=2", "https://news.ycombinator.com/news?p=3")
+page1, page2, page3 = HTTPX.get("https://news.ycombinator.com/news", "https://news.ycombinator.com/news?p=2", "https://news.ycombinator.com/news?p=3")
 ```
 
 ## Installation
@@ -107,22 +107,22 @@ HTTPX.get(
 
 ```ruby
 response = HTTPX.get("https://www.google.com", params: { q: "me" })
-response = HTTPX.post("https://www.nghttp2.org/httpbin/post", form: {name: "John", age: "22"})
+response = HTTPX.post("https://www.nghttp2.org/httpbin/post", form: { name: "John", age: "22" })
 response = HTTPX.plugin(:basic_auth)
                 .basic_auth("user", "pass")
                 .get("https://www.google.com")
 
 # more complex client objects can be cached, and are thread-safe
-http = HTTPX.plugin(:expect).with(headers: { "x-pvt-token" => "TOKEN"})
+http = HTTPX.plugin(:expect).with(headers: { "x-pvt-token" => "TOKEN" })
 http.get("https://example.com") # the above options will apply
-http.post("https://example2.com",  form: {name: "John", age: "22"}) # same, plus the form POST body
+http.post("https://example2.com", form: { name: "John", age: "22" }) # same, plus the form POST body
 ```
 
 ### Lightweight
 
 It ships with most features published as a plugin, making vanilla `httpx` lightweight and dependency-free, while allowing you to "pay for what you use"
 
-The plugin system is similar to the ones used by [sequel](https://github.com/jeremyevans/sequel), [roda](https://github.com/jeremyevans/roda) or [shrine](https://github.com/janko-m/shrine).
+The plugin system is similar to the ones used by [sequel](https://github.com/jeremyevans/sequel), [roda](https://github.com/jeremyevans/roda) or [shrine](https://github.com/shrinerb/shrine).
 
 ### Advanced DNS features
 

data/doc/release_notes/1_2_0.md

@@ -0,0 +1,49 @@
+# 1.2.0
+
+## Features
+
+### `:ssrf_filter` plugin
+
+The `:ssrf_filter` p plugin prevents server-side request forgery attacks, by blocking requests to the internal network. This is useful when the URLs used to perform requests aren’t under the developer control (such as when they are inserted via a web application form).
+
+```ruby
+http = HTTPX.plugin(:ssrf_filter)
+
+# this works
+response = http.get("https://example.com")
+
+# this doesn't
+response = http.get("http://localhost:3002")
+response = http.get("http://[::1]:3002")
+response = http.get("http://169.254.169.254/latest/meta-data/")
+```
+
+More info under https://honeyryderchuck.gitlab.io/httpx/wiki/SSRF-Filter
+
+### `:callbacks` plugin
+
+The session callbacks introduced in v0.24.0 are in its own plugin. Older code will still work and emit a deprecation warning.
+
+More info under https://honeyryderchuck.gitlab.io/httpx/wiki/Callbacks
+
+### `:redirect_on` option for `:follow_redirects` plugin
+
+This option allows passing a callback which, when returning `false`, can interrupt the redirect loop.
+
+```ruby
+http = HTTPX.plugin(:follow_redirects).with(redirect_on: ->(location_uri) { BLACKLIST_HOSTS.include?(location_uri.host) ]
+```
+
+### `:close_on_handshake_timeout` timeout
+
+A new `:timeout` option, `:close_handshake_timeout`, is added, which monitors connection readiness when performing HTTP/2 connection termination handshake.
+
+## Improvements
+
+* Internal "eden connections" concept was removed, and connection objects are now kept-and-reused during the lifetime of a session, even when closed. This simplified connectio pool implementation and improved performance.
+* request using `:proxy` and `:retries` plugin enabled sessions will now retry on proxy connection establishment related errors.
+
+## Bugfixes
+
+* webmock adapter: mocked responses storing decoded payloads won't try to decode them again (fixes vcr/webmock integrations).
+* webmock adapter: fix issue related with making real requests over webmock-enabled connection.

data/lib/httpx/adapters/webmock.rb

@@ -41,7 +41,9 @@ module WebMock
           request.options.response_class.new(request,
                                              webmock_response.status[0],
                                              "2.0",
-                                             webmock_response.headers)
+                                             webmock_response.headers).tap do |res|
+            res.mocked = true
+          end
         end
 
         def build_error_response(request, exception)
@@ -50,16 +52,36 @@ module WebMock
       end
 
       module InstanceMethods
-        def build_connection(*)
+        def init_connection(*)
           connection = super
           connection.once(:unmock_connection) do
+            unless connection.addresses
+              connection.__send__(:callbacks)[:connect_error].clear
+              pool.__send__(:unregister_connection, connection)
+            end
             pool.__send__(:resolve_connection, connection)
-            pool.__send__(:unregister_connection, connection) unless connection.addresses
           end
           connection
         end
       end
 
+      module ResponseMethods
+        attr_accessor :mocked
+
+        def initialize(*)
+          super
+          @mocked = false
+        end
+      end
+
+      module ResponseBodyMethods
+        def decode_chunk(chunk)
+          return chunk if @response.mocked
+
+          super
+        end
+      end
+
       module ConnectionMethods
         def initialize(*)
           super

data/lib/httpx/altsvc.rb

@@ -4,6 +4,58 @@ require "strscan"
 
 module HTTPX
   module AltSvc
+    # makes connections able to accept requests destined to primary service.
+    module ConnectionMixin
+      using URIExtensions
+
+      def send(request)
+        request.headers["alt-used"] = @origin.authority if @parser && !@write_buffer.full? && match_altsvcs?(request.uri)
+
+        super
+      end
+
+      def match?(uri, options)
+        return false if !used? && (@state == :closing || @state == :closed)
+
+        match_altsvcs?(uri) && match_altsvc_options?(uri, options)
+      end
+
+      private
+
+      # checks if this is connection is an alternative service of
+      # +uri+
+      def match_altsvcs?(uri)
+        @origins.any? { |origin| altsvc_match?(uri, origin) } ||
+          AltSvc.cached_altsvc(@origin).any? do |altsvc|
+            origin = altsvc["origin"]
+            altsvc_match?(origin, uri.origin)
+          end
+      end
+
+      def match_altsvc_options?(uri, options)
+        return @options == options unless @options.ssl.all? do |k, v|
+          v == (k == :hostname ? uri.host : options.ssl[k])
+        end
+
+        @options.options_equals?(options, Options::REQUEST_BODY_IVARS + %i[@ssl])
+      end
+
+      def altsvc_match?(uri, other_uri)
+        other_uri = URI(other_uri)
+
+        uri.origin == other_uri.origin || begin
+          case uri.scheme
+          when "h2"
+            (other_uri.scheme == "https" || other_uri.scheme == "h2") &&
+              uri.host == other_uri.host &&
+              uri.port == other_uri.port
+          else
+            false
+          end
+        end
+      end
+    end
+
     @altsvc_mutex = Thread::Mutex.new
     @altsvcs = Hash.new { |h, k| h[k] = [] }
 
@@ -46,7 +98,7 @@ module HTTPX
 
       altsvc = response.headers["alt-svc"]
 
-      # https://tools.ietf.org/html/rfc7838#section-3
+      # https://datatracker.ietf.org/doc/html/rfc7838#section-3
       # A field value containing the special value "clear" indicates that the
       # origin requests all alternatives for that origin to be invalidated
       # (including those specified in the same response, in case of an
@@ -99,7 +151,10 @@ module HTTPX
     end
 
     def parse_altsvc_origin(alt_proto, alt_origin)
-      alt_scheme = parse_altsvc_scheme(alt_proto) or return
+      alt_scheme = parse_altsvc_scheme(alt_proto)
+
+      return unless alt_scheme
+
       alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
 
       URI.parse("#{alt_scheme}://#{alt_origin}")

data/lib/httpx/chainable.rb

@@ -10,19 +10,6 @@ module HTTPX
       MOD
     end
 
-    %i[
-      connection_opened connection_closed
-      request_error
-      request_started request_body_chunk request_completed
-      response_started response_body_chunk response_completed
-    ].each do |meth|
-      class_eval(<<-MOD, __FILE__, __LINE__ + 1)
-        def on_#{meth}(&blk)   # def on_connection_opened(&blk)
-          on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
-        end                    # end
-      MOD
-    end
-
     def request(*args, **options)
       branch(default_options).request(*args, **options)
     end
@@ -46,12 +33,6 @@ module HTTPX
       branch(default_options.merge(options), &blk)
     end
 
-    protected
-
-    def on(*args, &blk)
-      branch(default_options).on(*args, &blk)
-    end
-
     private
 
     def default_options
@@ -64,22 +45,52 @@ module HTTPX
       Session.new(options, &blk)
     end
 
-    def method_missing(meth, *args, **options)
-      return super unless meth =~ /\Awith_(.+)/
+    def method_missing(meth, *args, **options, &blk)
+      case meth
+      when /\Awith_(.+)/
 
-      option = Regexp.last_match(1)
+        option = Regexp.last_match(1)
 
-      return super unless option
+        return super unless option
 
-      with(option.to_sym => (args.first || options))
-    end
+        with(option.to_sym => args.first || options)
+      when /\Aon_(.+)/
+        callback = Regexp.last_match(1)
 
-    def respond_to_missing?(meth, *)
-      return super unless meth =~ /\Awith_(.+)/
+        return super unless %w[
+          connection_opened connection_closed
+          request_error
+          request_started request_body_chunk request_completed
+          response_started response_body_chunk response_completed
+        ].include?(callback)
 
-      option = Regexp.last_match(1)
+        warn "DEPRECATION WARNING: calling `.#{meth}` on plain HTTPX sessions is deprecated. " \
+             "Use HTTPX.plugin(:callbacks).#{meth} instead."
 
-      default_options.respond_to?(option) || super
+        plugin(:callbacks).__send__(meth, *args, **options, &blk)
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(meth, *)
+      case meth
+      when /\Awith_(.+)/
+        option = Regexp.last_match(1)
+
+        default_options.respond_to?(option) || super
+      when /\Aon_(.+)/
+        callback = Regexp.last_match(1)
+
+        %w[
+          connection_opened connection_closed
+          request_error
+          request_started request_body_chunk request_completed
+          response_started response_body_chunk response_completed
+        ].include?(callback) || super
+      else
+        super
+      end
     end
   end
 end

data/lib/httpx/connection/http1.rb

@@ -12,6 +12,8 @@ module HTTPX
 
     attr_reader :pending, :requests
 
+    attr_accessor :max_concurrent_requests
+
     def initialize(buffer, options)
       @options = Options.new(options)
       @max_concurrent_requests = @options.max_concurrent_requests || MAX_REQUESTS
@@ -47,6 +49,7 @@ module HTTPX
       @max_requests = @options.max_requests || MAX_REQUESTS
       @parser.reset!
       @handshake_completed = false
+      @pending.concat(@requests) unless @requests.empty?
     end
 
     def close
@@ -218,6 +221,7 @@ module HTTPX
     end
 
     def ping
+      reset
       emit(:reset)
       emit(:exhausted)
     end
@@ -262,6 +266,7 @@ module HTTPX
 
     def disable
       disable_pipelining
+      reset
       emit(:reset)
       throw(:called)
     end
@@ -292,29 +297,31 @@ module HTTPX
         request.body.chunk!
       end
 
-      connection = request.headers["connection"]
+      extra_headers = {}
 
-      connection ||= if request.persistent?
-        # when in a persistent connection, the request can't be at
-        # the edge of a renegotiation
-        if @requests.index(request) + 1 < @max_requests
-          "keep-alive"
-        else
-          "close"
-        end
-      else
-        # when it's not a persistent connection, it sets "Connection: close" always
-        # on the last request of the possible batch (either allowed max requests,
-        # or if smaller, the size of the batch itself)
-        requests_limit = [@max_requests, @requests.size].min
-        if request == @requests[requests_limit - 1]
-          "close"
+      unless request.headers.key?("connection")
+        connection_value = if request.persistent?
+          # when in a persistent connection, the request can't be at
+          # the edge of a renegotiation
+          if @requests.index(request) + 1 < @max_requests
+            "keep-alive"
+          else
+            "close"
+          end
         else
-          "keep-alive"
+          # when it's not a persistent connection, it sets "Connection: close" always
+          # on the last request of the possible batch (either allowed max requests,
+          # or if smaller, the size of the batch itself)
+          requests_limit = [@max_requests, @requests.size].min
+          if request == @requests[requests_limit - 1]
+            "close"
+          else
+            "keep-alive"
+          end
         end
-      end
 
-      extra_headers = { "connection" => connection }
+        extra_headers["connection"] = connection_value
+      end
       extra_headers["host"] = request.authority unless request.headers.key?("host")
       extra_headers
     end
@@ -370,12 +377,10 @@ module HTTPX
     end
 
     def join_headers2(headers)
-      buffer = "".b
       headers.each do |field, value|
-        buffer << "#{capitalized(field)}: #{value}" << CRLF
+        buffer = "#{capitalized(field)}: #{value}#{CRLF}"
         log(color: :yellow) { "<- HEADER: #{buffer.chomp}" }
         @buffer << buffer
-        buffer.clear
       end
     end
 

data/lib/httpx/connection/http2.rb

@@ -55,7 +55,7 @@ module HTTPX
         return :w
       end
 
-      unless (@connection.state == :connected && @handshake_completed)
+      unless @connection.state == :connected && @handshake_completed
         return @buffer.empty? ? :r : :rw
       end
 
@@ -73,8 +73,11 @@ module HTTPX
     end
 
     def close
-      @connection.goaway unless @connection.state == :closed
-      emit(:close)
+      unless @connection.state == :closed
+        @connection.goaway
+        emit(:timeout, @options.timeout[:close_handshake_timeout])
+      end
+      emit(:close, true)
     end
 
     def empty?
@@ -147,6 +150,7 @@ module HTTPX
 
     def send_pending
       while (request = @pending.shift)
+        # TODO: this request should go back to top of stack
         break unless send(request)
       end
     end

data/lib/httpx/connection.rb

@@ -47,11 +47,11 @@ module HTTPX
 
     attr_accessor :family
 
-    def initialize(type, uri, options)
-      @type = type
+    def initialize(uri, options)
       @origins = [uri.origin]
       @origin = Utils.to_uri(uri.origin)
       @options = Options.new(options)
+      @type = initialize_type(uri, @options)
       @window_size = @options.window_size
       @read_buffer = Buffer.new(@options.buffer_size)
       @write_buffer = Buffer.new(@options.buffer_size)
@@ -92,18 +92,14 @@ module HTTPX
     def match?(uri, options)
       return false if !used? && (@state == :closing || @state == :closed)
 
-      return false if exhausted?
-
       (
-        (
-          @origins.include?(uri.origin) &&
-          # if there is more than one origin to match, it means that this connection
-          # was the result of coalescing. To prevent blind trust in the case where the
-          # origin came from an ORIGIN frame, we're going to verify the hostname with the
-          # SSL certificate
-          (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
-        ) && @options == options
-      ) || (match_altsvcs?(uri) && match_altsvc_options?(uri, options))
+        @origins.include?(uri.origin) &&
+        # if there is more than one origin to match, it means that this connection
+        # was the result of coalescing. To prevent blind trust in the case where the
+        # origin came from an ORIGIN frame, we're going to verify the hostname with the
+        # SSL certificate
+        (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
+      ) && @options == options
     end
 
     def expired?
@@ -115,8 +111,6 @@ module HTTPX
     def mergeable?(connection)
       return false if @state == :closing || @state == :closed || !@io
 
-      return false if exhausted?
-
       return false unless connection.addresses
 
       (
@@ -139,7 +133,7 @@ module HTTPX
     end
 
     def create_idle(options = {})
-      self.class.new(@type, @origin, @options.merge(options))
+      self.class.new(@origin, @options.merge(options))
     end
 
     def merge(connection)
@@ -167,24 +161,6 @@ module HTTPX
       end
     end
 
-    # checks if this is connection is an alternative service of
-    # +uri+
-    def match_altsvcs?(uri)
-      @origins.any? { |origin| uri.altsvc_match?(origin) } ||
-        AltSvc.cached_altsvc(@origin).any? do |altsvc|
-          origin = altsvc["origin"]
-          origin.altsvc_match?(uri.origin)
-        end
-    end
-
-    def match_altsvc_options?(uri, options)
-      return @options == options unless @options.ssl[:hostname] == uri.host
-
-      dup_options = @options.merge(ssl: { hostname: nil })
-      dup_options.ssl.delete(:hostname)
-      dup_options == options
-    end
-
     def connecting?
       @state == :idle
     end
@@ -223,7 +199,6 @@ module HTTPX
       when :closing
         consume
         transition(:closed)
-        emit(:close)
       when :open
         consume
       end
@@ -236,24 +211,33 @@ module HTTPX
       @parser.close if @parser
     end
 
+    def terminate
+      @connected_at = nil if @state == :closed
+
+      close
+    end
+
     # bypasses the state machine to force closing of connections still connecting.
     # **only** used for Happy Eyeballs v2.
     def force_reset
       @state = :closing
       transition(:closed)
-      emit(:close)
     end
 
     def reset
+      return if @state == :closing || @state == :closed
+
       transition(:closing)
+      unless @write_buffer.empty?
+        # handshakes, try sending
+        consume
+        @write_buffer.clear
+      end
       transition(:closed)
-      emit(:close)
     end
 
     def send(request)
       if @parser && !@write_buffer.full?
-        request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
-
         if @response_received_at && @keep_alive_timeout &&
            Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
           # when pushing a request into an existing connection, we have to check whether there
@@ -319,10 +303,6 @@ module HTTPX
       transition(:open)
     end
 
-    def exhausted?
-      @parser && parser.exhausted?
-    end
-
     def consume
       return unless @io
 
@@ -497,34 +477,25 @@ module HTTPX
         request.emit(:promise, parser, stream)
       end
       parser.on(:exhausted) do
+        @pending.concat(parser.pending)
         emit(:exhausted)
       end
       parser.on(:origin) do |origin|
         @origins |= [origin]
       end
       parser.on(:close) do |force|
-        if @state != :closed
-          transition(:closing)
-          if force || @state == :idle
-            transition(:closed)
-            emit(:close)
-          end
+        if force
+          reset
+          emit(:terminate)
         end
       end
       parser.on(:close_handshake) do
         consume
       end
       parser.on(:reset) do
-        if parser.empty?
-          reset
-        else
-          transition(:closing)
-          transition(:closed)
-
-          @parser.reset if @parser
-          transition(:idle)
-          transition(:open)
-        end
+        @pending.concat(parser.pending) unless parser.empty?
+        reset
+        idling unless @pending.empty?
       end
       parser.on(:current_timeout) do
         @current_timeout = @timeout = parser.timeout
@@ -593,13 +564,14 @@ module HTTPX
       when :inactive
         return unless @state == :open
       when :closing
-        return unless @state == :open
+        return unless @state == :idle || @state == :open
 
       when :closed
         return unless @state == :closing
         return unless @write_buffer.empty?
 
         purge_after_closed
+        emit(:close) if @pending.empty?
       when :already_open
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
@@ -621,6 +593,19 @@ module HTTPX
       @timeout = nil
     end
 
+    def initialize_type(uri, options)
+      options.transport || begin
+        case uri.scheme
+        when "http"
+          "tcp"
+        when "https"
+          "ssl"
+        else
+          raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
+        end
+      end
+    end
+
     def build_socket(addrs = nil)
       case @type
       when "tcp"

data/lib/httpx/extensions.rb

@@ -54,21 +54,6 @@ module HTTPX
       def origin
         "#{scheme}://#{authority}"
       end unless URI::HTTP.method_defined?(:origin)
-
-      def altsvc_match?(uri)
-        uri = URI.parse(uri)
-
-        origin == uri.origin || begin
-          case scheme
-          when "h2"
-            (uri.scheme == "https" || uri.scheme == "h2") &&
-            host == uri.host &&
-            (port || default_port) == (uri.port || uri.default_port)
-          else
-            false
-          end
-        end
-      end
     end
   end
 end