httpx 0.9.0 → 0.10.0

data/lib/httpx/plugins/expect.rb

@@ -33,11 +33,8 @@ module HTTPX
           super
           return if @body.nil?
 
-          if (threshold = options.expect_threshold_size)
-            unless unbounded_body?
-              return if @body.bytesize < threshold
-            end
-          end
+          threshold = options.expect_threshold_size
+          return if threshold && !unbounded_body? && @body.bytesize < threshold
 
           @headers["expect"] = "100-continue"
         end
@@ -63,6 +60,7 @@ module HTTPX
           return unless response
 
           if response.status == 417 && request.headers.key?("expect")
+            response.close
             request.headers.delete("expect")
             request.transition(:idle)
             connection = find_connection(request, connections, options)

data/lib/httpx/plugins/follow_redirects.rb

@@ -59,8 +59,26 @@ module HTTPX
             return ErrorResponse.new(request, error, options)
           end
 
-          connection = find_connection(retry_request, connections, options)
-          connection.send(retry_request)
+          retry_after = response.headers["retry-after"]
+
+          if retry_after
+            # Servers send the "Retry-After" header field to indicate how long the
+            # user agent ought to wait before making a follow-up request.
+            # When sent with any 3xx (Redirection) response, Retry-After indicates
+            # the minimum time that the user agent is asked to wait before issuing
+            # the redirected request.
+            #
+            retry_after = Utils.parse_retry_after(retry_after)
+
+            log { "redirecting after #{retry_after} secs..." }
+            pool.after(retry_after) do
+              connection = find_connection(retry_request, connections, options)
+              connection.send(retry_request)
+            end
+          else
+            connection = find_connection(retry_request, connections, options)
+            connection.send(retry_request)
+          end
           nil
         end
 

data/lib/httpx/plugins/h2c.rb

@@ -73,7 +73,7 @@ module HTTPX
 
           # clean up data left behind in the buffer, if the server started
           # sending frames
-          data = response.to_s
+          data = response.read
           @connection << data
         end
       end

data/lib/httpx/plugins/multipart.rb

@@ -29,14 +29,6 @@ module HTTPX
           def bytesize
             @raw.content_length
           end
-
-          def force_encoding(*args)
-            @raw.to_s.force_encoding(*args)
-          end
-
-          def to_str
-            @raw.to_s
-          end
         end
 
         def encode(form)

data/lib/httpx/plugins/persistent.rb

@@ -19,7 +19,12 @@ module HTTPX
     #
     module Persistent
       def self.load_dependencies(klass)
-        klass.plugin(:retries, max_retries: 1, retry_change_requests: true)
+        max_retries = if klass.default_options.respond_to?(:max_retries)
+          [klass.default_options.max_retries, 1].max
+        else
+          1
+        end
+        klass.plugin(:retries, max_retries: max_retries, retry_change_requests: true)
       end
 
       def self.extra_options(options)

data/lib/httpx/plugins/proxy/socks4.rb

@@ -91,6 +91,8 @@ module HTTPX
         end
 
         module Packet
+          using(RegexpExtensions) unless Regexp.method_defined?(:match?)
+
           module_function
 
           def connect(parameters, uri)
@@ -101,7 +103,7 @@ module HTTPX
 
               packet << [ip.to_i].pack("N")
             rescue IPAddr::InvalidAddressError
-              if parameters.uri.scheme =~ /^socks4a?$/
+              if /^socks4a?$/.match?(parameters.uri.scheme)
                 # resolv defaults to IPv4, and socks4 doesn't support IPv6 otherwise
                 ip = IPAddr.new(Resolv.getaddress(uri.host))
                 packet << [ip.to_i].pack("N")

data/lib/httpx/plugins/rate_limiter.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for retrying requests when the request:
+    #
+    # * is rate limited;
+    # * when the server is unavailable (503);
+    # * when a 3xx request comes with a "retry-after" value
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/RateLimiter
+    #
+    module RateLimiter
+      class << self
+        RATE_LIMIT_CODES = [429, 503].freeze
+
+        def load_dependencies(klass)
+          klass.plugin(:retries,
+                       retry_change_requests: true,
+                       retry_on: method(:retry_on_rate_limited_response),
+                       retry_after: method(:retry_after_rate_limit))
+        end
+
+        def retry_on_rate_limited_response(response)
+          status = response.status
+
+          RATE_LIMIT_CODES.include?(status)
+        end
+
+        # Servers send the "Retry-After" header field to indicate how long the
+        # user agent ought to wait before making a follow-up request.  When
+        # sent with a 503 (Service Unavailable) response, Retry-After indicates
+        # how long the service is expected to be unavailable to the client.
+        # When sent with any 3xx (Redirection) response, Retry-After indicates
+        # the minimum time that the user agent is asked to wait before issuing
+        # the redirected request.
+        #
+        def retry_after_rate_limit(_, response)
+          retry_after = response.headers["retry-after"]
+
+          return unless retry_after
+
+          Utils.parse_retry_after(retry_after)
+        end
+      end
+    end
+
+    register_plugin :rate_limiter, RateLimiter
+  end
+end

data/lib/httpx/plugins/retries.rb

@@ -75,14 +75,15 @@ module HTTPX
                )
                # rubocop:enable Style/MultilineTernaryOperator
              )
-
+            response.close if response.respond_to?(:close)
             request.retries -= 1
             log { "failed to get response, #{request.retries} tries to go..." }
             request.transition(:idle)
 
             retry_after = options.retry_after
+            retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
+
             if retry_after
-              retry_after = retry_after.call(request) if retry_after.respond_to?(:call)
 
               log { "retrying after #{retry_after} secs..." }
               pool.after(retry_after) do

data/lib/httpx/plugins/stream.rb

@@ -7,27 +7,123 @@ module HTTPX
     #
     module Stream
       module InstanceMethods
-        def stream
-          headers("accept" => "text/event-stream",
-                  "cache-control" => "no-cache")
+        private
+
+        def request(*args, stream: false, **options)
+          return super(*args, **options) unless stream
+
+          requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
+
+          raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
+
+          StreamResponse.new(requests.first, self)
         end
       end
 
+      module RequestMethods
+        attr_accessor :stream
+      end
+
       module ResponseMethods
-        def complete?
-          super ||
-            stream? &&
-              @stream_complete
+        def stream
+          @request.stream
+        end
+      end
+
+      module ResponseBodyMethods
+        def initialize(*, **)
+          super
+          @stream = @response.stream
+        end
+
+        def write(chunk)
+          return super unless @stream
+
+          @stream.on_chunk(chunk.to_s.dup)
+        end
+
+        private
+
+        def transition(*)
+          return if @stream
+
+          super
+        end
+      end
+
+      class StreamResponse
+        def initialize(request, session)
+          @request = request
+          @session = session
+          @options = @request.options
+        end
+
+        def each(&block)
+          return enum_for(__method__) unless block_given?
+
+          raise Error, "response already streamed" if @response
+
+          @request.stream = self
+
+          begin
+            @on_chunk = block
+
+            response.raise_for_status
+            response.close
+          ensure
+            @on_chunk = nil
+          end
+        end
+
+        def each_line
+          return enum_for(__method__) unless block_given?
+
+          line = +""
+
+          each do |chunk|
+            line << chunk
+
+            while (idx = line.index("\n"))
+              yield line.byteslice(0..idx - 1)
+
+              line = line.byteslice(idx + 1..-1)
+            end
+          end
+        end
+
+        # This is a ghost method. It's to be used ONLY internally, when processing streams
+        def on_chunk(chunk)
+          raise NoMethodError unless @on_chunk
+
+          @on_chunk.call(chunk)
+        end
+
+        # :nocov:
+        def inspect
+          "#<StreamResponse:#{object_id}>"
+        end
+        # :nocov:
+
+        def to_s
+          response.to_s
+        end
+
+        private
+
+        def response
+          @response ||= @session.__send__(:send_requests, @request, @options).first
         end
 
-        def stream?
-          @headers["content-type"].start_with?("text/event-stream")
+        def respond_to_missing?(*args)
+          @options.response_class.respond_to?(*args) || super
         end
 
-        def <<(data)
-          res = super
-          @stream_complete = true if String(data).end_with?("\n\n")
-          res
+        def method_missing(meth, *args, &block)
+          if @options.response_class.public_method_defined?(meth)
+            response.__send__(meth, *args, &block)
+          else
+            super
+          end
         end
       end
     end

data/lib/httpx/pool.rb

@@ -37,20 +37,22 @@ module HTTPX
 
         @timers.fire
       end
-    rescue Interrupt
-      @connections.each(&:reset)
-      raise
     rescue StandardError => e
       @connections.each do |connection|
         connection.emit(:error, e)
       end
+    rescue Exception # rubocop:disable Lint/RescueException
+      @connections.each(&:reset)
+      raise
     end
 
     def close(connections = @connections)
+      return if connections.empty?
+
       @timers.cancel
       connections = connections.reject(&:inflight?)
       connections.each(&:close)
-      next_tick until connections.none? { |c| @connections.include?(c) }
+      next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
       @resolvers.each_value do |resolver|
         resolver.close unless resolver.closed?
       end if @connections.empty?
@@ -166,7 +168,6 @@ module HTTPX
         resolver.on(:error, &method(:on_resolver_error))
         resolver.on(:close) { on_resolver_close(resolver) }
         resolver
-      # rubocop: disable Layout/RescueEnsureAlignment
       rescue ArgumentError
         # this block is here because of an error which happens on CI from time to time
         warn "tried resolver: #{resolver_type}"
@@ -174,7 +175,6 @@ module HTTPX
         warn "new: #{resolver_type.method(:new).source_location}"
         raise
       end
-      # rubocop: enable Layout/RescueEnsureAlignment
     end
   end
 end

data/lib/httpx/request.rb

@@ -33,11 +33,7 @@ module HTTPX
 
     USER_AGENT = "httpx.rb/#{VERSION}"
 
-    attr_reader :verb, :uri, :headers, :body, :state
-
-    attr_reader :options, :response
-
-    def_delegator :@body, :<<
+    attr_reader :verb, :uri, :headers, :body, :state, :options, :response
 
     def_delegator :@body, :empty?
 
@@ -45,7 +41,7 @@ module HTTPX
 
     def initialize(verb, uri, options = {})
       @verb    = verb.to_s.downcase.to_sym
-      @uri     = URI(uri.to_s)
+      @uri     = Utils.uri(uri)
       @options = Options.new(options)
 
       raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)
@@ -64,17 +60,15 @@ module HTTPX
       :w
     end
 
-    # :nocov:
     if RUBY_VERSION < "2.2"
-      # rubocop: disable Lint/UriEscapeUnescape:
+      URIParser = URI::DEFAULT_PARSER
+
       def initialize_with_escape(verb, uri, options = {})
-        initialize_without_escape(verb, URI.escape(uri.to_s), options)
+        initialize_without_escape(verb, URIParser.escape(uri.to_s), options)
       end
       alias_method :initialize_without_escape, :initialize
       alias_method :initialize, :initialize_with_escape
-      # rubocop: enable Lint/UriEscapeUnescape:
     end
-    # :nocov:
 
     def merge_headers(h)
       @headers = @headers.merge(h)
@@ -180,19 +174,13 @@ module HTTPX
         return true if @body.nil?
         return false if chunked?
 
-        bytesize.zero?
+        @body.bytesize.zero?
       end
 
       def bytesize
         return 0 if @body.nil?
 
-        if @body.respond_to?(:bytesize)
-          @body.bytesize
-        elsif @body.respond_to?(:size)
-          @body.size
-        else
-          raise Error, "cannot determine size of body: #{@body.inspect}"
-        end
+        @body.bytesize
       end
 
       def stream(body)

data/lib/httpx/resolver/https.rb

@@ -94,7 +94,12 @@ module HTTPX
     def resolve(connection = @connections.first, hostname = nil)
       return if @building_connection
 
-      hostname = hostname || @queries.key(connection) || connection.origin.host
+      hostname ||= @queries.key(connection)
+
+      if hostname.nil?
+        hostname = connection.origin.host
+        log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+      end
       type = @_record_types[hostname].first
       log { "resolver: query #{type} for #{hostname}" }
       begin
@@ -206,7 +211,7 @@ module HTTPX
       case response.headers["content-type"]
       when "application/dns-json",
            "application/json",
-           %r{^application\/x\-javascript} # because google...
+           %r{^application/x-javascript} # because google...
         payload = JSON.parse(response.to_s)
         payload["Answer"]
       when "application/dns-udpwireformat",

data/lib/httpx/resolver/native.rb

@@ -7,6 +7,7 @@ module HTTPX
   class Resolver::Native
     extend Forwardable
     include Resolver::ResolverMixin
+    using URIExtensions
 
     RESOLVE_TIMEOUT = 5
     RECORD_TYPES = {
@@ -168,7 +169,6 @@ module HTTPX
         siz = @io.read(wsize, @read_buffer)
         return unless siz && siz.positive?
 
-        log { "resolver: READ: #{siz} bytes..." }
         parse(@read_buffer)
         return if @state == :closed
       end
@@ -181,7 +181,6 @@ module HTTPX
         siz = @io.write(@write_buffer)
         return unless siz && siz.positive?
 
-        log { "resolver: WRITE: #{siz} bytes..." }
         return if @state == :closed
       end
     end
@@ -237,7 +236,12 @@ module HTTPX
       raise Error, "no URI to resolve" unless connection
       return unless @write_buffer.empty?
 
-      hostname = hostname || @queries.key(connection) || connection.origin.host
+      hostname ||= @queries.key(connection)
+
+      if hostname.nil?
+        hostname = connection.origin.host
+        log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+      end
       @queries[hostname] = connection
       type = @_record_types[hostname].first
       log { "resolver: query #{type} for #{hostname}" }