httpx 0.9.0 → 0.11.1

data/lib/httpx/plugins/multipart/mime_type_detector.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::Multipart
+    module MimeTypeDetector
+      module_function
+
+      DEFAULT_MIMETYPE = "application/octet-stream"
+
+      # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
+      if defined?(MIME::Types)
+
+        def call(_file, filename)
+          mime = MIME::Types.of(filename).first
+          mime.content_type if mime
+        end
+
+      elsif defined?(MimeMagic)
+
+        def call(file, *)
+          mime = MimeMagic.by_magic(file)
+          mime.type if mime
+        end
+
+      elsif system("which file", out: File::NULL)
+        require "open3"
+
+        def call(file, *)
+          return if file.eof? # file command returns "application/x-empty" for empty files
+
+          Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
+            begin
+              ::IO.copy_stream(file, stdin.binmode)
+            rescue Errno::EPIPE
+            end
+            file.rewind
+            stdin.close
+
+            status = thread.value
+
+            # call to file command failed
+            if status.nil? || !status.success?
+              $stderr.print(stderr.read)
+            else
+
+              output = stdout.read.strip
+
+              if output.include?("cannot open")
+                $stderr.print(output)
+              else
+                output
+              end
+            end
+          end
+        end
+
+      else
+
+        def call(*); end
+
+      end
+    end
+  end
+end

data/lib/httpx/plugins/multipart/part.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::Multipart
+    module Part
+      module_function
+
+      def call(value)
+        # take out specialized objects of the way
+        if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
+          return [value, value.content_type, value.filename]
+        end
+
+        content_type = filename = nil
+
+        if value.is_a?(Hash)
+          content_type = value[:content_type]
+          filename = value[:filename]
+          value = value[:body]
+        end
+
+        value = value.open(:binmode => true) if value.is_a?(Pathname)
+
+        if value.is_a?(File)
+          filename ||= File.basename(value.path)
+          content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
+          [value, content_type, filename]
+        else
+          [StringIO.new(value.to_s), "text/plain"]
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/persistent.rb

@@ -19,7 +19,12 @@ module HTTPX
     #
     module Persistent
       def self.load_dependencies(klass)
-        klass.plugin(:retries, max_retries: 1, retry_change_requests: true)
+        max_retries = if klass.default_options.respond_to?(:max_retries)
+          [klass.default_options.max_retries, 1].max
+        else
+          1
+        end
+        klass.plugin(:retries, max_retries: max_retries, retry_change_requests: true)
       end
 
       def self.extra_options(options)

data/lib/httpx/plugins/proxy.rb

@@ -121,8 +121,7 @@ module HTTPX
         def fetch_response(request, connections, options)
           response = super
           if response.is_a?(ErrorResponse) &&
-             # either it was a timeout error connecting, or it was a proxy error
-             PROXY_ERRORS.any? { |ex| response.error.is_a?(ex) } && !@_proxy_uris.empty?
+             __proxy_error?(response) && !@_proxy_uris.empty?
             @_proxy_uris.shift
             log { "failed connecting to proxy, trying next..." }
             request.transition(:idle)
@@ -139,6 +138,21 @@ module HTTPX
 
           super
         end
+
+        def __proxy_error?(response)
+          error = response.error
+          case error
+          when ResolveError
+            # failed resolving proxy domain
+            proxy_uri = error.connection.options.proxy.uri
+            proxy_uri.to_s == @_proxy_uris.first
+          when *PROXY_ERRORS
+            # timeout errors connecting to proxy
+            true
+          else
+            false
+          end
+        end
       end
 
       module ConnectionMethods

data/lib/httpx/plugins/proxy/socks4.rb

@@ -10,7 +10,7 @@ module HTTPX
       module Socks4
         VERSION = 4
         CONNECT = 1
-        GRANTED = 90
+        GRANTED = 0x5A
         PROTOCOLS = %w[socks4 socks4a].freeze
 
         Error = Socks4Error
@@ -95,21 +95,21 @@ module HTTPX
 
           def connect(parameters, uri)
             packet = [VERSION, CONNECT, uri.port].pack("CCn")
-            begin
-              ip = IPAddr.new(uri.host)
-              raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
-
-              packet << [ip.to_i].pack("N")
-            rescue IPAddr::InvalidAddressError
-              if parameters.uri.scheme =~ /^socks4a?$/
-                # resolv defaults to IPv4, and socks4 doesn't support IPv6 otherwise
-                ip = IPAddr.new(Resolv.getaddress(uri.host))
-                packet << [ip.to_i].pack("N")
-              else
-                packet << "\x0\x0\x0\x1" << "\x7\x0" << uri.host
+
+            case parameters.uri.scheme
+            when "socks4"
+              socks_host = uri.host
+              begin
+                ip = IPAddr.new(socks_host)
+                packet << ip.hton
+              rescue IPAddr::InvalidAddressError
+                socks_host = Resolv.getaddress(socks_host)
+                retry
               end
+              packet << [parameters.username].pack("Z*")
+            when "socks4a"
+              packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
             end
-            packet << [parameters.username].pack("Z*")
             packet
           end
         end

data/lib/httpx/plugins/proxy/socks5.rb

@@ -159,9 +159,10 @@ module HTTPX
             packet = [VERSION, CONNECT, 0].pack("C*")
             begin
               ip = IPAddr.new(uri.host)
-              raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
 
-              packet << [IPV4, ip.to_i].pack("CN")
+              ipcode = ip.ipv6? ? IPV6 : IPV4
+
+              packet << [ipcode].pack("C") << ip.hton
             rescue IPAddr::InvalidAddressError
               packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
             end

data/lib/httpx/plugins/push_promise.rb

@@ -70,8 +70,8 @@ module HTTPX
           request.transition(:done)
           response = request.response
           response.mark_as_pushed!
-          stream.on(:data, &parser.method(:on_stream_data).curry[stream, request])
-          stream.on(:close, &parser.method(:on_stream_close).curry[stream, request])
+          stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
+          stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
         end
       end
     end

data/lib/httpx/plugins/rate_limiter.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for retrying requests when the request:
+    #
+    # * is rate limited;
+    # * when the server is unavailable (503);
+    # * when a 3xx request comes with a "retry-after" value
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/RateLimiter
+    #
+    module RateLimiter
+      class << self
+        RATE_LIMIT_CODES = [429, 503].freeze
+
+        def load_dependencies(klass)
+          klass.plugin(:retries,
+                       retry_change_requests: true,
+                       retry_on: method(:retry_on_rate_limited_response),
+                       retry_after: method(:retry_after_rate_limit))
+        end
+
+        def retry_on_rate_limited_response(response)
+          status = response.status
+
+          RATE_LIMIT_CODES.include?(status)
+        end
+
+        # Servers send the "Retry-After" header field to indicate how long the
+        # user agent ought to wait before making a follow-up request.  When
+        # sent with a 503 (Service Unavailable) response, Retry-After indicates
+        # how long the service is expected to be unavailable to the client.
+        # When sent with any 3xx (Redirection) response, Retry-After indicates
+        # the minimum time that the user agent is asked to wait before issuing
+        # the redirected request.
+        #
+        def retry_after_rate_limit(_, response)
+          retry_after = response.headers["retry-after"]
+
+          return unless retry_after
+
+          Utils.parse_retry_after(retry_after)
+        end
+      end
+    end
+
+    register_plugin :rate_limiter, RateLimiter
+  end
+end

data/lib/httpx/plugins/retries.rb

@@ -75,14 +75,15 @@ module HTTPX
                )
                # rubocop:enable Style/MultilineTernaryOperator
              )
-
+            response.close if response.respond_to?(:close)
             request.retries -= 1
             log { "failed to get response, #{request.retries} tries to go..." }
             request.transition(:idle)
 
             retry_after = options.retry_after
+            retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
+
             if retry_after
-              retry_after = retry_after.call(request) if retry_after.respond_to?(:call)
 
               log { "retrying after #{retry_after} secs..." }
               pool.after(retry_after) do

data/lib/httpx/plugins/stream.rb

@@ -7,27 +7,123 @@ module HTTPX
     #
     module Stream
       module InstanceMethods
-        def stream
-          headers("accept" => "text/event-stream",
-                  "cache-control" => "no-cache")
+        private
+
+        def request(*args, stream: false, **options)
+          return super(*args, **options) unless stream
+
+          requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
+
+          raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
+
+          StreamResponse.new(requests.first, self)
         end
       end
 
+      module RequestMethods
+        attr_accessor :stream
+      end
+
       module ResponseMethods
-        def complete?
-          super ||
-            stream? &&
-              @stream_complete
+        def stream
+          @request.stream
+        end
+      end
+
+      module ResponseBodyMethods
+        def initialize(*, **)
+          super
+          @stream = @response.stream
+        end
+
+        def write(chunk)
+          return super unless @stream
+
+          @stream.on_chunk(chunk.to_s.dup)
+        end
+
+        private
+
+        def transition(*)
+          return if @stream
+
+          super
+        end
+      end
+
+      class StreamResponse
+        def initialize(request, session)
+          @request = request
+          @session = session
+          @options = @request.options
+        end
+
+        def each(&block)
+          return enum_for(__method__) unless block_given?
+
+          raise Error, "response already streamed" if @response
+
+          @request.stream = self
+
+          begin
+            @on_chunk = block
+
+            response.raise_for_status
+            response.close
+          ensure
+            @on_chunk = nil
+          end
+        end
+
+        def each_line
+          return enum_for(__method__) unless block_given?
+
+          line = +""
+
+          each do |chunk|
+            line << chunk
+
+            while (idx = line.index("\n"))
+              yield line.byteslice(0..idx - 1)
+
+              line = line.byteslice(idx + 1..-1)
+            end
+          end
+        end
+
+        # This is a ghost method. It's to be used ONLY internally, when processing streams
+        def on_chunk(chunk)
+          raise NoMethodError unless @on_chunk
+
+          @on_chunk.call(chunk)
+        end
+
+        # :nocov:
+        def inspect
+          "#<StreamResponse:#{object_id}>"
+        end
+        # :nocov:
+
+        def to_s
+          response.to_s
+        end
+
+        private
+
+        def response
+          @response ||= @session.__send__(:send_requests, @request, @options).first
         end
 
-        def stream?
-          @headers["content-type"].start_with?("text/event-stream")
+        def respond_to_missing?(*args)
+          @options.response_class.respond_to?(*args) || super
         end
 
-        def <<(data)
-          res = super
-          @stream_complete = true if String(data).end_with?("\n\n")
-          res
+        def method_missing(meth, *args, &block)
+          if @options.response_class.public_method_defined?(meth)
+            response.__send__(meth, *args, &block)
+          else
+            super
+          end
         end
       end
     end

data/lib/httpx/pool.rb

@@ -37,20 +37,22 @@ module HTTPX
 
         @timers.fire
       end
-    rescue Interrupt
-      @connections.each(&:reset)
-      raise
     rescue StandardError => e
       @connections.each do |connection|
         connection.emit(:error, e)
       end
+    rescue Exception # rubocop:disable Lint/RescueException
+      @connections.each(&:reset)
+      raise
     end
 
     def close(connections = @connections)
+      return if connections.empty?
+
       @timers.cancel
       connections = connections.reject(&:inflight?)
       connections.each(&:close)
-      next_tick until connections.none? { |c| @connections.include?(c) }
+      next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
       @resolvers.each_value do |resolver|
         resolver.close unless resolver.closed?
       end if @connections.empty?
@@ -106,10 +108,10 @@ module HTTPX
       end
     end
 
-    def on_resolver_error(ch, error)
-      ch.emit(:error, error)
+    def on_resolver_error(connection, error)
+      connection.emit(:error, error)
       # must remove connection by hand, hasn't been started yet
-      unregister_connection(ch)
+      unregister_connection(connection)
     end
 
     def on_resolver_close(resolver)
@@ -142,12 +144,12 @@ module HTTPX
       @connected_connections -= 1
     end
 
-    def coalesce_connections(ch1, ch2)
-      if ch1.coalescable?(ch2)
-        ch1.merge(ch2)
-        @connections.delete(ch2)
+    def coalesce_connections(conn1, conn2)
+      if conn1.coalescable?(conn2)
+        conn1.merge(conn2)
+        @connections.delete(conn2)
       else
-        register_connection(ch2)
+        register_connection(conn2)
       end
     end
 
@@ -166,15 +168,7 @@ module HTTPX
         resolver.on(:error, &method(:on_resolver_error))
         resolver.on(:close) { on_resolver_close(resolver) }
         resolver
-      # rubocop: disable Layout/RescueEnsureAlignment
-      rescue ArgumentError
-        # this block is here because of an error which happens on CI from time to time
-        warn "tried resolver: #{resolver_type}"
-        warn "initialize: #{resolver_type.instance_method(:initialize).source_location}"
-        warn "new: #{resolver_type.method(:new).source_location}"
-        raise
       end
-      # rubocop: enable Layout/RescueEnsureAlignment
     end
   end
 end