httpx 0.18.2 → 0.18.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2f0ac68d969d4462c03c28fa818fa055e4e3aabad5a599197df6cc912274673
-  data.tar.gz: '069b5100a14183e79a968f4bb69363304ce046d098d1283f934568ea0fc23e06'
+  metadata.gz: b6784fbd393b5caed888e6c4d1a421cde9c7aca49f35baaafbf13f10a68e9d02
+  data.tar.gz: ddd5bb8793fa72b19e462b411f5eaaaf553f30d589a68dfdde64224c49d902f6
 SHA512:
-  metadata.gz: bddcf91c4cd851204ff789798bc8745e6e7f2b39face3ae77c83de4aa8be4de278f24a0ce1405f5e142f2f358fbf78472a5a816ae315ac43574c81adb6b82838
-  data.tar.gz: f9d992536ae6e464833dbe05c6a884ba373f84d90e42657bd18bab9fc69288561361041563dbe26dee92e9b9f337ee57d7eb7b9f7586abe705905184cd57f9e3
+  metadata.gz: 949f18741f10eaa57463b8764569e69cdfb1ca6f4eefa30bbdcf142cbbd8b61b360d9bd78898a816a4ebd9e15d4ffd1c0d8282033a459826919ce8132d5b105f
+  data.tar.gz: e4430eea967c9f5628d51e6150bc4f63e704eaed2bd61fda191f3539a3f102e00932cc8f63ddb735c33bbf21e2694bb23fed5eaea588159e71fa6f484016c150

data/doc/release_notes/0_18_3.md

@@ -0,0 +1,7 @@
+# 0.18.3
+
+## Bugfixes
+
+* request bodies eager-loaded from enumerables yield duped partial chunks.
+
+An error was observed while looking at webmock integration, where requests formed via the multipart plugin were returning an empty string as body. The issue was caused by an optimization on multipart encoder, which reuses the same buffer when reading chunks. Unfortunately, these cannot be yielded the same way via IO.copy_stream, as the same (cleared) buffer will be used to generate the eager-loaded body chunks.

data/doc/release_notes/0_18_4.md

@@ -0,0 +1,14 @@
+# 0.18.4
+
+## Improvements
+
+* faraday adapter: added support for `#on_data` callback in order to support [faraday streaming](https://lostisland.github.io/faraday/usage/streaming).
+
+* multipart plugin: removed support for file mime type detection using `mime-types`. The reasoning behind it was that `mime-types` uses the filename, which is a very inaccurate detection strategy (ex: an mp4 video will be identified as `application/mp4`, instead of the correct `video/mp4`).
+* multipart plugin: supported for file mime type detection using `marcel` and `filemagic` was added. Both use the magic header bytes, which is a more accurate strategy for file type detection.
+
+## Bugfixes
+
+* webmock adapter has been reimplemented to work with `httpx` plugins (such as the `:retries` plugin). Some other fixes were applied to make it work better under `vcr` (a common `webmock` extension).
+
+* fixed the URI-related bug which was making requests stall under ruby 3.1 (still not officially testing against it).

data/doc/release_notes/0_18_5.md

@@ -0,0 +1,10 @@
+# 0.18.5
+
+## Improvements
+
+* ruby 3.1 is now officially supported.
+* when user sets a `Host` header for an HTTP/2 request, this will be used in the `:authority` HTTP/2 pseudo-header, instead of silently ignored (mimicking what "curl" does).
+
+## Bugfixes
+
+* fixed "throw outside of catch block" error happening when pipelining requests on an HTTP/1 connnection and resulting in a timeout.

data/doc/release_notes/0_18_6.md

@@ -0,0 +1,5 @@
+# 0.18.6
+
+## Bugfixes
+
+* multipart plugin: fixed missing constant in `filemagic` integration.

data/lib/httpx/adapters/faraday.rb

@@ -21,6 +21,17 @@ module Faraday
       end
       # :nocov:
 
+      unless Faraday::RequestOptions.method_defined?(:stream_response?)
+        module RequestOptionsExtensions
+          refine Faraday::RequestOptions do
+            def stream_response?
+              false
+            end
+          end
+        end
+        using RequestOptionsExtensions
+      end
+
       module RequestMixin
         using ::HTTPX::HashExtensions
 
@@ -64,6 +75,27 @@ module Faraday
 
       include RequestMixin
 
+      module OnDataPlugin
+        module RequestMethods
+          attr_writer :response_on_data
+
+          def response=(response)
+            super
+            response.body.on_data = @response_on_data
+          end
+        end
+
+        module ResponseBodyMethods
+          attr_writer :on_data
+
+          def write(chunk)
+            return super unless @on_data
+
+            @on_data.call(chunk, chunk.bytesize)
+          end
+        end
+      end
+
       class Session < ::HTTPX::Session
         plugin(:compression)
         plugin(:persistent)
@@ -137,15 +169,21 @@ module Faraday
         end
 
         def run
-          requests = @handlers.map { |handler| build_request(handler.env) }
           env = @handlers.last.env
 
-          proxy_options = { uri: env.request.proxy }
-
           session = @session.with(options_from_env(env))
-          session = session.plugin(:proxy).with(proxy: proxy_options) if env.request.proxy
+          session = session.plugin(:proxy).with(proxy: { uri: env.request.proxy }) if env.request.proxy
+          session = session.plugin(OnDataPlugin) if env.request.stream_response?
+
+          requests = @handlers.map { |handler| session.build_request(*build_request(handler.env)) }
+
+          if env.request.stream_response?
+            requests.each do |request|
+              request.response_on_data = env.request.on_data
+            end
+          end
 
-          responses = session.request(requests)
+          responses = session.request(*requests)
           Array(responses).each_with_index do |response, index|
             handler = @handlers[index]
             handler.on_response.call(response)
@@ -179,11 +217,15 @@ module Faraday
           return handler
         end
 
-        meth, uri, request_options = build_request(env)
-
         session = @session.with(options_from_env(env))
-        session = session.plugin(:proxy).with(proxy: proxy_options) if env.request.proxy
-        response = session.__send__(meth, uri, **request_options)
+        session = session.plugin(:proxy).with(proxy: { uri: env.request.proxy }) if env.request.proxy
+        session = session.plugin(OnDataPlugin) if env.request.stream_response?
+
+        request = session.build_request(*build_request(env))
+
+        request.response_on_data = env.request.on_data if env.request.stream_response?
+
+        response = session.request(request)
         response.raise_for_status unless response.is_a?(::HTTPX::Response)
         save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
           response_headers.merge!(response.headers)

data/lib/httpx/adapters/webmock.rb

@@ -16,56 +16,8 @@ module WebMock
     # Requests are "hijacked" at the session, before they're distributed to a connection.
     #
     module Plugin
-      module InstanceMethods
-        private
-
-        def send_requests(*requests)
-          request_signatures = requests.map do |request|
-            request_signature = _build_webmock_request_signature(request)
-            WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
-            request_signature
-          end
-
-          responses = request_signatures.map do |request_signature|
-            WebMock::StubRegistry.instance.response_for_request(request_signature)
-          end
-
-          real_requests = {}
-
-          requests.each_with_index.each_with_object([request_signatures, responses]) do |(request, idx), (sig_reqs, mock_responses)|
-            if (webmock_response = mock_responses[idx])
-              mock_responses[idx] = _build_from_webmock_response(request, webmock_response)
-              WebMock::CallbackRegistry.invoke_callbacks({ lib: :httpx }, sig_reqs[idx], webmock_response)
-              log { "mocking #{request.uri} with #{mock_responses[idx].inspect}" }
-            elsif WebMock.net_connect_allowed?(sig_reqs[idx].uri)
-              log { "performing #{request.uri}" }
-              real_requests[request] = idx
-            else
-              raise WebMock::NetConnectNotAllowedError, sig_reqs[idx]
-            end
-          end
-
-          unless real_requests.empty?
-            reqs = real_requests.keys
-            reqs.zip(super(*reqs)).each do |req, res|
-              idx = real_requests[req]
-
-              if WebMock::CallbackRegistry.any_callbacks?
-                webmock_response = _build_webmock_response(req, res)
-                WebMock::CallbackRegistry.invoke_callbacks(
-                  { lib: :httpx, real_request: true }, request_signatures[idx],
-                  webmock_response
-                )
-              end
-
-              responses[idx] = res
-            end
-          end
-
-          responses
-        end
-
-        def _build_webmock_request_signature(request)
+      class << self
+        def build_webmock_request_signature(request)
           uri = WebMock::Util::URI.heuristic_parse(request.uri)
           uri.path = uri.normalized_path.gsub("[^:]//", "/")
 
@@ -77,7 +29,7 @@ module WebMock
           )
         end
 
-        def _build_webmock_response(_request, response)
+        def build_webmock_response(_request, response)
           webmock_response = WebMock::Response.new
           webmock_response.status = [response.status, HTTP_REASONS[response.status]]
           webmock_response.body = response.body.to_s
@@ -85,10 +37,10 @@ module WebMock
           webmock_response
         end
 
-        def _build_from_webmock_response(request, webmock_response)
-          return _build_error_response(request, HTTPX::TimeoutError.new(1, "Timed out")) if webmock_response.should_timeout
+        def build_from_webmock_response(request, webmock_response)
+          return build_error_response(request, HTTPX::TimeoutError.new(1, "Timed out")) if webmock_response.should_timeout
 
-          return _build_error_response(request, webmock_response.exception) if webmock_response.exception
+          return build_error_response(request, webmock_response.exception) if webmock_response.exception
 
           response = request.options.response_class.new(request,
                                                         webmock_response.status[0],
@@ -98,10 +50,70 @@ module WebMock
           response
         end
 
-        def _build_error_response(request, exception)
+        def build_error_response(request, exception)
           HTTPX::ErrorResponse.new(request, exception, request.options)
         end
       end
+
+      module InstanceMethods
+        def build_connection(*)
+          connection = super
+          connection.once(:unmock_connection) do
+            pool.__send__(:resolve_connection, connection)
+            pool.__send__(:unregister_connection, connection) unless connection.addresses
+          end
+          connection
+        end
+      end
+
+      module ConnectionMethods
+        def initialize(*)
+          super
+          @mocked = true
+        end
+
+        def open?
+          return true if @mocked
+
+          super
+        end
+
+        def interests
+          return if @mocked
+
+          super
+        end
+
+        def send(request)
+          request_signature = Plugin.build_webmock_request_signature(request)
+          WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
+
+          if (mock_response = WebMock::StubRegistry.instance.response_for_request(request_signature))
+            response = Plugin.build_from_webmock_response(request, mock_response)
+            WebMock::CallbackRegistry.invoke_callbacks({ lib: :httpx }, request_signature, mock_response)
+            log { "mocking #{request.uri} with #{mock_response.inspect}" }
+            request.response = response
+            request.emit(:response, response)
+          elsif WebMock.net_connect_allowed?(request_signature.uri)
+            if WebMock::CallbackRegistry.any_callbacks?
+              request.on(:response) do |resp|
+                unless resp.is_a?(HTTPX::ErrorResponse)
+                  webmock_response = Plugin.build_webmock_response(request, resp)
+                  WebMock::CallbackRegistry.invoke_callbacks(
+                    { lib: :httpx, real_request: true }, request_signature,
+                    webmock_response
+                  )
+                end
+              end
+            end
+            @mocked = false
+            emit(:unmock_connection, self)
+            super
+          else
+            raise WebMock::NetConnectNotAllowedError, request_signature
+          end
+        end
+      end
     end
 
     class HttpxAdapter < HttpLibAdapter
@@ -109,12 +121,12 @@ module WebMock
 
       class << self
         def enable!
-          @original_session = ::HTTPX::Session
+          @original_session = HTTPX::Session
 
-          webmock_session = ::HTTPX.plugin(Plugin)
+          webmock_session = HTTPX.plugin(Plugin)
 
-          ::HTTPX.send(:remove_const, :Session)
-          ::HTTPX.send(:const_set, :Session, webmock_session.class)
+          HTTPX.send(:remove_const, :Session)
+          HTTPX.send(:const_set, :Session, webmock_session.class)
         end
 
         def disable!

data/lib/httpx/altsvc.rb

@@ -70,7 +70,7 @@ module HTTPX
 
       scanner = StringScanner.new(altsvc)
       until scanner.eos?
-        alt_origin = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
+        alt_service = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
 
         alt_params = []
         loop do
@@ -80,29 +80,45 @@ module HTTPX
           break if scanner.eos? || scanner.scan(/ *, */)
         end
         alt_params = Hash[alt_params.map { |field| field.split("=") }]
-        yield(parse_altsvc_origin(alt_origin), alt_params)
+
+        alt_proto, alt_authority = alt_service.split("=")
+        alt_origin = parse_altsvc_origin(alt_proto, alt_authority)
+        return unless alt_origin
+
+        yield(alt_origin, alt_params.merge("proto" => alt_proto))
+      end
+    end
+
+    def parse_altsvc_scheme(alt_proto)
+      case alt_proto
+      when "h2c"
+        "http"
+      when "h2"
+        "https"
       end
     end
 
     # :nocov:
     if RUBY_VERSION < "2.2"
-      def parse_altsvc_origin(alt_origin)
-        alt_proto, alt_origin = alt_origin.split("=")
+      def parse_altsvc_origin(alt_proto, alt_origin)
+        alt_scheme = parse_altsvc_scheme(alt_proto) or return
+
         alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
         if alt_origin.start_with?(":")
-          alt_origin = "#{alt_proto}://dummy#{alt_origin}"
+          alt_origin = "#{alt_scheme}://dummy#{alt_origin}"
           uri = URI.parse(alt_origin)
           uri.host = nil
           uri
         else
-          URI.parse("#{alt_proto}://#{alt_origin}")
+          URI.parse("#{alt_scheme}://#{alt_origin}")
         end
       end
     else
-      def parse_altsvc_origin(alt_origin)
-        alt_proto, alt_origin = alt_origin.split("=")
+      def parse_altsvc_origin(alt_proto, alt_origin)
+        alt_scheme = parse_altsvc_scheme(alt_proto) or return
         alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
-        URI.parse("#{alt_proto}://#{alt_origin}")
+
+        URI.parse("#{alt_scheme}://#{alt_origin}")
       end
     end
     # :nocov:

data/lib/httpx/connection/http1.rb

@@ -184,7 +184,7 @@ module HTTPX
       end
 
       if @pipelining
-        disable
+        catch(:called) { disable }
       else
         @requests.each do |request|
           emit(:error, request, ex)

data/lib/httpx/connection/http2.rb

@@ -220,6 +220,12 @@ module HTTPX
 
     def join_headers(stream, request)
       extra_headers = set_protocol_headers(request)
+
+      if request.headers.key?("host")
+        log { "forbidden \"host\" header found (#{request.headers["host"]}), will use it as authority..." }
+        extra_headers[":authority"] = request.headers["host"]
+      end
+
       log(level: 1, color: :yellow) do
         request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
       end

data/lib/httpx/io/udp.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "socket"
 require "ipaddr"
 
 module HTTPX

data/lib/httpx/plugins/multipart/mime_type_detector.rb

@@ -8,11 +8,21 @@ module HTTPX
       DEFAULT_MIMETYPE = "application/octet-stream"
 
       # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
-      if defined?(MIME::Types)
+      if defined?(FileMagic)
+        MAGIC_NUMBER = 256 * 1024
 
-        def call(_file, filename)
-          mime = MIME::Types.of(filename).first
-          mime.content_type if mime
+        def call(file, _)
+          return nil if file.eof? # FileMagic returns "application/x-empty" for empty files
+
+          FileMagic.open(FileMagic::MAGIC_MIME_TYPE) do |filemagic|
+            filemagic.buffer(file.read(MAGIC_NUMBER))
+          end
+        end
+      elsif defined?(Marcel)
+        def call(file, filename)
+          return nil if file.eof? # marcel returns "application/octet-stream" for empty files
+
+          Marcel::MimeType.for(file, name: filename)
         end
 
       elsif defined?(MimeMagic)

data/lib/httpx/plugins/proxy.rb

@@ -138,10 +138,20 @@ module HTTPX
           error = response.error
           case error
           when NativeResolveError
+            return false unless @_proxy_uris && !@_proxy_uris.empty?
+
+            proxy_uri = URI(@_proxy_uris.first)
+
+            origin = error.connection.origin
+
             # failed resolving proxy domain
-            error.connection.origin.to_s == @_proxy_uris.first
+            origin.host == proxy_uri.host && origin.port == proxy_uri.port
           when ResolveError
-            error.message.end_with?(@_proxy_uris.first)
+            return false unless @_proxy_uris && !@_proxy_uris.empty?
+
+            proxy_uri = URI(@_proxy_uris.first)
+
+            error.message.end_with?(proxy_uri.to_s)
           when *PROXY_ERRORS
             # timeout errors connecting to proxy
             true
@@ -160,7 +170,9 @@ module HTTPX
 
           # redefining the connection origin as the proxy's URI,
           # as this will be used as the tcp peer ip.
-          @origin = URI(@options.proxy.uri.origin)
+          proxy_uri = URI(@options.proxy.uri)
+          @origin.host = proxy_uri.host
+          @origin.port = proxy_uri.port
         end
 
         def match?(uri, options)

data/lib/httpx/request.rb

@@ -285,7 +285,7 @@ module HTTPX
       end
 
       def write(data)
-        @block.call(data)
+        @block.call(data.dup)
         data.bytesize
       end
     end

data/lib/httpx/session.rb

@@ -163,7 +163,7 @@ module HTTPX
         case uri.scheme
         when "http"
           "tcp"
-        when "https", "h2"
+        when "https"
           "ssl"
         else
           raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.18.2"
+  VERSION = "0.18.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httpx
 version: !ruby/object:Gem::Version
-  version: 0.18.2
+  version: 0.18.6
 platform: ruby
 authors:
 - Tiago Cardoso
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-09 00:00:00.000000000 Z
+date: 2022-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http-2-next
@@ -65,6 +65,10 @@ extra_rdoc_files:
 - doc/release_notes/0_18_0.md
 - doc/release_notes/0_18_1.md
 - doc/release_notes/0_18_2.md
+- doc/release_notes/0_18_3.md
+- doc/release_notes/0_18_4.md
+- doc/release_notes/0_18_5.md
+- doc/release_notes/0_18_6.md
 - doc/release_notes/0_1_0.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md
@@ -123,6 +127,10 @@ files:
 - doc/release_notes/0_18_0.md
 - doc/release_notes/0_18_1.md
 - doc/release_notes/0_18_2.md
+- doc/release_notes/0_18_3.md
+- doc/release_notes/0_18_4.md
+- doc/release_notes/0_18_5.md
+- doc/release_notes/0_18_6.md
 - doc/release_notes/0_1_0.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md
@@ -320,7 +328,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: HTTPX, to the future, and beyond