httpx 0.7.0 → 0.10.0

data/lib/httpx/plugins/proxy/socks5.rb

@@ -31,6 +31,10 @@ module HTTPX
             end
           end
 
+          def connecting?
+            super || @state == :authenticating || @state == :negotiating
+          end
+
           private
 
           def transition(nextstate)
@@ -60,7 +64,7 @@ module HTTPX
 
               @parser = nil
             end
-            log(level: 1, label: "SOCKS5: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
+            log(level: 1) { "SOCKS5: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
             super
           end
 

data/lib/httpx/plugins/push_promise.rb

@@ -43,9 +43,9 @@ module HTTPX
         end
 
         def __on_promise_request(parser, stream, h)
-          log(level: 1, label: "#{stream.id}: ") do
+          log(level: 1) do
             # :nocov:
-            h.map { |k, v| "-> PROMISE HEADER: #{k}: #{v}" }.join("\n")
+            h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
             # :nocov:
           end
           headers = @options.headers_class.new(h)

data/lib/httpx/plugins/rate_limiter.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for retrying requests when the request:
+    #
+    # * is rate limited;
+    # * when the server is unavailable (503);
+    # * when a 3xx request comes with a "retry-after" value
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/RateLimiter
+    #
+    module RateLimiter
+      class << self
+        RATE_LIMIT_CODES = [429, 503].freeze
+
+        def load_dependencies(klass)
+          klass.plugin(:retries,
+                       retry_change_requests: true,
+                       retry_on: method(:retry_on_rate_limited_response),
+                       retry_after: method(:retry_after_rate_limit))
+        end
+
+        def retry_on_rate_limited_response(response)
+          status = response.status
+
+          RATE_LIMIT_CODES.include?(status)
+        end
+
+        # Servers send the "Retry-After" header field to indicate how long the
+        # user agent ought to wait before making a follow-up request.  When
+        # sent with a 503 (Service Unavailable) response, Retry-After indicates
+        # how long the service is expected to be unavailable to the client.
+        # When sent with any 3xx (Redirection) response, Retry-After indicates
+        # the minimum time that the user agent is asked to wait before issuing
+        # the redirected request.
+        #
+        def retry_after_rate_limit(_, response)
+          retry_after = response.headers["retry-after"]
+
+          return unless retry_after
+
+          Utils.parse_retry_after(retry_after)
+        end
+      end
+    end
+
+    register_plugin :rate_limiter, RateLimiter
+  end
+end

data/lib/httpx/plugins/retries.rb

@@ -63,20 +63,27 @@ module HTTPX
         def fetch_response(request, connections, options)
           response = super
 
-          retry_on = options.retry_on
-
-          if response.is_a?(ErrorResponse) &&
+          if response &&
              request.retries.positive? &&
              __repeatable_request?(request, options) &&
-             __retryable_error?(response.error) &&
-             (!retry_on || retry_on.call(response))
+             (
+               # rubocop:disable Style/MultilineTernaryOperator
+               options.retry_on ?
+               options.retry_on.call(response) :
+               (
+                 response.is_a?(ErrorResponse) && __retryable_error?(response.error)
+               )
+               # rubocop:enable Style/MultilineTernaryOperator
+             )
+            response.close if response.respond_to?(:close)
             request.retries -= 1
             log { "failed to get response, #{request.retries} tries to go..." }
             request.transition(:idle)
 
             retry_after = options.retry_after
+            retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
+
             if retry_after
-              retry_after = retry_after.call(request) if retry_after.respond_to?(:call)
 
               log { "retrying after #{retry_after} secs..." }
               pool.after(retry_after) do

data/lib/httpx/plugins/stream.rb

@@ -7,27 +7,123 @@ module HTTPX
     #
     module Stream
       module InstanceMethods
-        def stream
-          headers("accept" => "text/event-stream",
-                  "cache-control" => "no-cache")
+        private
+
+        def request(*args, stream: false, **options)
+          return super(*args, **options) unless stream
+
+          requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
+
+          raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
+
+          StreamResponse.new(requests.first, self)
         end
       end
 
+      module RequestMethods
+        attr_accessor :stream
+      end
+
       module ResponseMethods
-        def complete?
-          super ||
-            stream? &&
-              @stream_complete
+        def stream
+          @request.stream
+        end
+      end
+
+      module ResponseBodyMethods
+        def initialize(*, **)
+          super
+          @stream = @response.stream
+        end
+
+        def write(chunk)
+          return super unless @stream
+
+          @stream.on_chunk(chunk.to_s.dup)
+        end
+
+        private
+
+        def transition(*)
+          return if @stream
+
+          super
+        end
+      end
+
+      class StreamResponse
+        def initialize(request, session)
+          @request = request
+          @session = session
+          @options = @request.options
+        end
+
+        def each(&block)
+          return enum_for(__method__) unless block_given?
+
+          raise Error, "response already streamed" if @response
+
+          @request.stream = self
+
+          begin
+            @on_chunk = block
+
+            response.raise_for_status
+            response.close
+          ensure
+            @on_chunk = nil
+          end
+        end
+
+        def each_line
+          return enum_for(__method__) unless block_given?
+
+          line = +""
+
+          each do |chunk|
+            line << chunk
+
+            while (idx = line.index("\n"))
+              yield line.byteslice(0..idx - 1)
+
+              line = line.byteslice(idx + 1..-1)
+            end
+          end
+        end
+
+        # This is a ghost method. It's to be used ONLY internally, when processing streams
+        def on_chunk(chunk)
+          raise NoMethodError unless @on_chunk
+
+          @on_chunk.call(chunk)
+        end
+
+        # :nocov:
+        def inspect
+          "#<StreamResponse:#{object_id}>"
+        end
+        # :nocov:
+
+        def to_s
+          response.to_s
+        end
+
+        private
+
+        def response
+          @response ||= @session.__send__(:send_requests, @request, @options).first
         end
 
-        def stream?
-          @headers["content-type"].start_with?("text/event-stream")
+        def respond_to_missing?(*args)
+          @options.response_class.respond_to?(*args) || super
         end
 
-        def <<(data)
-          res = super
-          @stream_complete = true if String(data).end_with?("\n\n")
-          res
+        def method_missing(meth, *args, &block)
+          if @options.response_class.public_method_defined?(meth)
+            response.__send__(meth, *args, &block)
+          else
+            super
+          end
         end
       end
     end

data/lib/httpx/pool.rb

@@ -14,7 +14,7 @@ module HTTPX
 
     def initialize
       @resolvers = {}
-      @_resolver_monitors = {}
+      @_resolver_ios = {}
       @timers = Timers::Group.new
       @selector = Selector.new
       @connections = []
@@ -28,28 +28,31 @@ module HTTPX
     def next_tick
       catch(:jump_tick) do
         timeout = [next_timeout, @timers.wait_interval].compact.min
-        if timeout.negative?
+        if timeout && timeout.negative?
           @timers.fire
           throw(:jump_tick)
         end
 
-        @selector.select(timeout) do |monitor|
-          monitor.io.call
-          monitor.interests = monitor.io.interests
-        end
+        @selector.select(timeout, &:call)
+
         @timers.fire
       end
     rescue StandardError => e
       @connections.each do |connection|
         connection.emit(:error, e)
       end
+    rescue Exception # rubocop:disable Lint/RescueException
+      @connections.each(&:reset)
+      raise
     end
 
     def close(connections = @connections)
+      return if connections.empty?
+
       @timers.cancel
       connections = connections.reject(&:inflight?)
       connections.each(&:close)
-      next_tick until connections.none? { |c| @connections.include?(c) }
+      next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
       @resolvers.each_value do |resolver|
         resolver.close unless resolver.closed?
       end if @connections.empty?
@@ -86,7 +89,7 @@ module HTTPX
       resolver << connection
       return if resolver.empty?
 
-      @_resolver_monitors[resolver] ||= @selector.register(resolver, :w)
+      @_resolver_ios[resolver] ||= @selector.register(resolver)
     end
 
     def on_resolver_connection(connection)
@@ -118,8 +121,7 @@ module HTTPX
       @resolvers.delete(resolver_type)
 
       @selector.deregister(resolver)
-      monitor = @_resolver_monitors.delete(resolver)
-      monitor.close if monitor
+      @_resolver_ios.delete(resolver)
       resolver.close unless resolver.closed?
     end
 
@@ -128,10 +130,8 @@ module HTTPX
         # if open, an IO was passed upstream, therefore
         # consider it connected already.
         @connected_connections += 1
-        @selector.register(connection, :rw)
-      else
-        @selector.register(connection, :w)
       end
+      @selector.register(connection)
       connection.on(:close) do
         unregister_connection(connection)
       end
@@ -168,7 +168,6 @@ module HTTPX
         resolver.on(:error, &method(:on_resolver_error))
         resolver.on(:close) { on_resolver_close(resolver) }
         resolver
-      # rubocop: disable Layout/RescueEnsureAlignment
       rescue ArgumentError
         # this block is here because of an error which happens on CI from time to time
         warn "tried resolver: #{resolver_type}"
@@ -176,7 +175,6 @@ module HTTPX
         warn "new: #{resolver_type.method(:new).source_location}"
         raise
       end
-      # rubocop: enable Layout/RescueEnsureAlignment
     end
   end
 end

data/lib/httpx/registry.rb

@@ -64,7 +64,8 @@ module HTTPX
 
         case handler
         when Symbol, String
-          const_get(handler)
+          obj = const_get(handler)
+          @registry[tag] = obj
         else
           handler
         end

data/lib/httpx/request.rb

@@ -33,11 +33,7 @@ module HTTPX
 
     USER_AGENT = "httpx.rb/#{VERSION}"
 
-    attr_reader :verb, :uri, :headers, :body, :state
-
-    attr_reader :options, :response
-
-    def_delegator :@body, :<<
+    attr_reader :verb, :uri, :headers, :body, :state, :options, :response
 
     def_delegator :@body, :empty?
 
@@ -45,7 +41,7 @@ module HTTPX
 
     def initialize(verb, uri, options = {})
       @verb    = verb.to_s.downcase.to_sym
-      @uri     = URI(uri.to_s)
+      @uri     = Utils.uri(uri)
       @options = Options.new(options)
 
       raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)
@@ -58,17 +54,21 @@ module HTTPX
       @state = :idle
     end
 
-    # :nocov:
+    def interests
+      return :r if @state == :done || @state == :expect
+
+      :w
+    end
+
     if RUBY_VERSION < "2.2"
-      # rubocop: disable Lint/UriEscapeUnescape:
+      URIParser = URI::DEFAULT_PARSER
+
       def initialize_with_escape(verb, uri, options = {})
-        initialize_without_escape(verb, URI.escape(uri.to_s), options)
+        initialize_without_escape(verb, URIParser.escape(uri.to_s), options)
       end
       alias_method :initialize_without_escape, :initialize
       alias_method :initialize, :initialize_with_escape
-      # rubocop: enable Lint/UriEscapeUnescape:
     end
-    # :nocov:
 
     def merge_headers(h)
       @headers = @headers.merge(h)
@@ -174,19 +174,13 @@ module HTTPX
         return true if @body.nil?
         return false if chunked?
 
-        bytesize.zero?
+        @body.bytesize.zero?
       end
 
       def bytesize
         return 0 if @body.nil?
 
-        if @body.respond_to?(:bytesize)
-          @body.bytesize
-        elsif @body.respond_to?(:size)
-          @body.size
-        else
-          raise Error, "cannot determine size of body: #{@body.inspect}"
-        end
+        @body.bytesize
       end
 
       def stream(body)
@@ -219,6 +213,7 @@ module HTTPX
       case nextstate
       when :idle
         @response = nil
+        @drainer = nil
       when :headers
         return unless @state == :idle
       when :body

data/lib/httpx/resolver.rb

@@ -1,19 +1,18 @@
 # frozen_string_literal: true
 
 require "resolv"
+require "httpx/resolver/resolver_mixin"
+require "httpx/resolver/system"
+require "httpx/resolver/native"
+require "httpx/resolver/https"
 
 module HTTPX
   module Resolver
-    autoload :ResolverMixin, "httpx/resolver/resolver_mixin"
-    autoload :System, "httpx/resolver/system"
-    autoload :Native, "httpx/resolver/native"
-    autoload :HTTPS, "httpx/resolver/https"
-
     extend Registry
 
-    register :system, :System
-    register :native, :Native
-    register :https,  :HTTPS
+    register :system, System
+    register :native, Native
+    register :https,  HTTPS
 
     @lookup_mutex = Mutex.new
     @lookups = Hash.new { |h, k| h[k] = [] }