httpx 0.8.0 → 0.10.1

data/lib/httpx/plugins/cookies/set_cookie_parser.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "strscan"
+require "time"
+
+module HTTPX
+  module Plugins::Cookies
+    module SetCookieParser
+      using(RegexpExtensions) unless Regexp.method_defined?(:match?)
+
+      # Whitespace.
+      RE_WSP = /[ \t]+/.freeze
+
+      # A pattern that matches a cookie name or attribute name which may
+      # be empty, capturing trailing whitespace.
+      RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
+
+      RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
+
+      # A pattern that matches the comma in a (typically date) value.
+      RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
+
+      module_function
+
+      def scan_dquoted(scanner)
+        s = +""
+
+        until scanner.eos?
+          break if scanner.skip(/"/)
+
+          if scanner.skip(/\\/)
+            s << scanner.getch
+          elsif scanner.scan(/[^"\\]+/)
+            s << scanner.matched
+          end
+        end
+
+        s
+      end
+
+      def scan_value(scanner, comma_as_separator = false)
+        value = +""
+
+        until scanner.eos?
+          if scanner.scan(/[^,;"]+/)
+            value << scanner.matched
+          elsif scanner.skip(/"/)
+            # RFC 6265 2.2
+            # A cookie-value may be DQUOTE'd.
+            value << scan_dquoted(scanner)
+          elsif scanner.check(/;/)
+            break
+          elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
+            break
+          else
+            value << scanner.getch
+          end
+        end
+
+        value.rstrip!
+        value
+      end
+
+      def scan_name_value(scanner, comma_as_separator = false)
+        name = scanner.scan(RE_NAME)
+        name.rstrip! if name
+
+        if scanner.skip(/=/)
+          value = scan_value(scanner, comma_as_separator)
+        else
+          scan_value(scanner, comma_as_separator)
+          value = nil
+        end
+        [name, value]
+      end
+
+      def call(set_cookie)
+        scanner = StringScanner.new(set_cookie)
+
+        # RFC 6265 4.1.1 & 5.2
+        until scanner.eos?
+          start = scanner.pos
+          len = nil
+
+          scanner.skip(RE_WSP)
+
+          name, value = scan_name_value(scanner, true)
+          value = nil if name.empty?
+
+          attrs = {}
+
+          until scanner.eos?
+            if scanner.skip(/,/)
+              # The comma is used as separator for concatenating multiple
+              # values of a header.
+              len = (scanner.pos - 1) - start
+              break
+            elsif scanner.skip(/;/)
+              scanner.skip(RE_WSP)
+
+              aname, avalue = scan_name_value(scanner, true)
+
+              next if aname.empty? || value.nil?
+
+              aname.downcase!
+
+              case aname
+              when "expires"
+                # RFC 6265 5.2.1
+                (avalue &&= Time.httpdate(avalue)) || next
+              when "max-age"
+                # RFC 6265 5.2.2
+                next unless /\A-?\d+\z/.match?(avalue)
+
+                avalue = Integer(avalue)
+              when "domain"
+                # RFC 6265 5.2.3
+                # An empty value SHOULD be ignored.
+                next if avalue.nil? || avalue.empty?
+              when "path"
+                # RFC 6265 5.2.4
+                # A relative path must be ignored rather than normalizing it
+                # to "/".
+                next unless avalue.start_with?("/")
+              when "secure", "httponly"
+                # RFC 6265 5.2.5, 5.2.6
+                avalue = true
+              end
+              attrs[aname] = avalue
+            end
+          end
+
+          len ||= scanner.pos - start
+
+          next if len > Cookie::MAX_LENGTH
+
+          yield(name, value, attrs) if name && !name.empty? && value
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/expect.rb

@@ -18,14 +18,24 @@ module HTTPX
 
             seconds
           end
+
+          def_option(:expect_threshold_size) do |bytes|
+            bytes = Integer(bytes)
+            raise Error, ":expect_threshold_size must be positive" unless bytes.positive?
+
+            bytes
+          end
         end.new(options).merge(expect_timeout: EXPECT_TIMEOUT)
       end
 
       module RequestBodyMethods
-        def initialize(*)
+        def initialize(*, options)
           super
           return if @body.nil?
 
+          threshold = options.expect_threshold_size
+          return if threshold && !unbounded_body? && @body.bytesize < threshold
+
           @headers["expect"] = "100-continue"
         end
       end
@@ -50,6 +60,7 @@ module HTTPX
           return unless response
 
           if response.status == 417 && request.headers.key?("expect")
+            response.close
             request.headers.delete("expect")
             request.transition(:idle)
             connection = find_connection(request, connections, options)

data/lib/httpx/plugins/follow_redirects.rb

@@ -59,8 +59,26 @@ module HTTPX
             return ErrorResponse.new(request, error, options)
           end
 
-          connection = find_connection(retry_request, connections, options)
-          connection.send(retry_request)
+          retry_after = response.headers["retry-after"]
+
+          if retry_after
+            # Servers send the "Retry-After" header field to indicate how long the
+            # user agent ought to wait before making a follow-up request.
+            # When sent with any 3xx (Redirection) response, Retry-After indicates
+            # the minimum time that the user agent is asked to wait before issuing
+            # the redirected request.
+            #
+            retry_after = Utils.parse_retry_after(retry_after)
+
+            log { "redirecting after #{retry_after} secs..." }
+            pool.after(retry_after) do
+              connection = find_connection(retry_request, connections, options)
+              connection.send(retry_request)
+            end
+          else
+            connection = find_connection(retry_request, connections, options)
+            connection.send(retry_request)
+          end
           nil
         end
 

data/lib/httpx/plugins/h2c.rb

@@ -73,7 +73,7 @@ module HTTPX
 
           # clean up data left behind in the buffer, if the server started
           # sending frames
-          data = response.to_s
+          data = response.read
           @connection << data
         end
       end

data/lib/httpx/plugins/multipart.rb

@@ -23,19 +23,25 @@ module HTTPX
           def_delegator :@raw, :read
 
           def initialize(form)
-            @raw = HTTP::FormData.create(form)
+            @raw = if multipart?(form)
+              HTTP::FormData::Multipart.new(Hash[*form.map { |k, v| Transcoder.enum_for(:normalize_keys, k, v).to_a }])
+            else
+              HTTP::FormData::Urlencoded.new(form, :encoder => Transcoder::Form.method(:encode))
+            end
           end
 
           def bytesize
             @raw.content_length
           end
 
-          def force_encoding(*args)
-            @raw.to_s.force_encoding(*args)
-          end
+          private
 
-          def to_str
-            @raw.to_s
+          def multipart?(data)
+            data.any? do |_, v|
+              v.is_a?(HTTP::FormData::Part) ||
+                (v.respond_to?(:to_ary) && v.to_ary.any? { |e| e.is_a?(HTTP::FormData::Part) }) ||
+                (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| e.is_a?(HTTP::FormData::Part) })
+            end
           end
         end
 

data/lib/httpx/plugins/persistent.rb

@@ -19,7 +19,12 @@ module HTTPX
     #
     module Persistent
       def self.load_dependencies(klass)
-        klass.plugin(:retries, max_retries: 1, retry_change_requests: true)
+        max_retries = if klass.default_options.respond_to?(:max_retries)
+          [klass.default_options.max_retries, 1].max
+        else
+          1
+        end
+        klass.plugin(:retries, max_retries: max_retries, retry_change_requests: true)
       end
 
       def self.extra_options(options)

data/lib/httpx/plugins/proxy.rb

@@ -121,8 +121,7 @@ module HTTPX
         def fetch_response(request, connections, options)
           response = super
           if response.is_a?(ErrorResponse) &&
-             # either it was a timeout error connecting, or it was a proxy error
-             PROXY_ERRORS.any? { |ex| response.error.is_a?(ex) } && !@_proxy_uris.empty?
+             __proxy_error?(response) && !@_proxy_uris.empty?
             @_proxy_uris.shift
             log { "failed connecting to proxy, trying next..." }
             request.transition(:idle)
@@ -139,6 +138,21 @@ module HTTPX
 
           super
         end
+
+        def __proxy_error?(response)
+          error = response.error
+          case error
+          when ResolveError
+            # failed resolving proxy domain
+            proxy_uri = error.connection.options.proxy.uri
+            proxy_uri.to_s == @_proxy_uris.first
+          when *PROXY_ERRORS
+            # timeout errors connecting to proxy
+            true
+          else
+            false
+          end
+        end
       end
 
       module ConnectionMethods

data/lib/httpx/plugins/proxy/socks4.rb

@@ -10,7 +10,7 @@ module HTTPX
       module Socks4
         VERSION = 4
         CONNECT = 1
-        GRANTED = 90
+        GRANTED = 0x5A
         PROTOCOLS = %w[socks4 socks4a].freeze
 
         Error = Socks4Error
@@ -95,21 +95,21 @@ module HTTPX
 
           def connect(parameters, uri)
             packet = [VERSION, CONNECT, uri.port].pack("CCn")
-            begin
-              ip = IPAddr.new(uri.host)
-              raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
-
-              packet << [ip.to_i].pack("N")
-            rescue IPAddr::InvalidAddressError
-              if parameters.uri.scheme =~ /^socks4a?$/
-                # resolv defaults to IPv4, and socks4 doesn't support IPv6 otherwise
-                ip = IPAddr.new(Resolv.getaddress(uri.host))
-                packet << [ip.to_i].pack("N")
-              else
-                packet << "\x0\x0\x0\x1" << "\x7\x0" << uri.host
+
+            case parameters.uri.scheme
+            when "socks4"
+              socks_host = uri.host
+              begin
+                ip = IPAddr.new(socks_host)
+                packet << ip.hton
+              rescue IPAddr::InvalidAddressError
+                socks_host = Resolv.getaddress(socks_host)
+                retry
               end
+              packet << [parameters.username].pack("Z*")
+            when "socks4a"
+              packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
             end
-            packet << [parameters.username].pack("Z*")
             packet
           end
         end

data/lib/httpx/plugins/rate_limiter.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for retrying requests when the request:
+    #
+    # * is rate limited;
+    # * when the server is unavailable (503);
+    # * when a 3xx request comes with a "retry-after" value
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/RateLimiter
+    #
+    module RateLimiter
+      class << self
+        RATE_LIMIT_CODES = [429, 503].freeze
+
+        def load_dependencies(klass)
+          klass.plugin(:retries,
+                       retry_change_requests: true,
+                       retry_on: method(:retry_on_rate_limited_response),
+                       retry_after: method(:retry_after_rate_limit))
+        end
+
+        def retry_on_rate_limited_response(response)
+          status = response.status
+
+          RATE_LIMIT_CODES.include?(status)
+        end
+
+        # Servers send the "Retry-After" header field to indicate how long the
+        # user agent ought to wait before making a follow-up request.  When
+        # sent with a 503 (Service Unavailable) response, Retry-After indicates
+        # how long the service is expected to be unavailable to the client.
+        # When sent with any 3xx (Redirection) response, Retry-After indicates
+        # the minimum time that the user agent is asked to wait before issuing
+        # the redirected request.
+        #
+        def retry_after_rate_limit(_, response)
+          retry_after = response.headers["retry-after"]
+
+          return unless retry_after
+
+          Utils.parse_retry_after(retry_after)
+        end
+      end
+    end
+
+    register_plugin :rate_limiter, RateLimiter
+  end
+end

data/lib/httpx/plugins/retries.rb

@@ -75,14 +75,15 @@ module HTTPX
                )
                # rubocop:enable Style/MultilineTernaryOperator
              )
-
+            response.close if response.respond_to?(:close)
             request.retries -= 1
             log { "failed to get response, #{request.retries} tries to go..." }
             request.transition(:idle)
 
             retry_after = options.retry_after
+            retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
+
             if retry_after
-              retry_after = retry_after.call(request) if retry_after.respond_to?(:call)
 
               log { "retrying after #{retry_after} secs..." }
               pool.after(retry_after) do

data/lib/httpx/plugins/stream.rb

@@ -7,27 +7,123 @@ module HTTPX
     #
     module Stream
       module InstanceMethods
-        def stream
-          headers("accept" => "text/event-stream",
-                  "cache-control" => "no-cache")
+        private
+
+        def request(*args, stream: false, **options)
+          return super(*args, **options) unless stream
+
+          requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
+
+          raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
+
+          StreamResponse.new(requests.first, self)
         end
       end
 
+      module RequestMethods
+        attr_accessor :stream
+      end
+
       module ResponseMethods
-        def complete?
-          super ||
-            stream? &&
-              @stream_complete
+        def stream
+          @request.stream
+        end
+      end
+
+      module ResponseBodyMethods
+        def initialize(*, **)
+          super
+          @stream = @response.stream
+        end
+
+        def write(chunk)
+          return super unless @stream
+
+          @stream.on_chunk(chunk.to_s.dup)
+        end
+
+        private
+
+        def transition(*)
+          return if @stream
+
+          super
+        end
+      end
+
+      class StreamResponse
+        def initialize(request, session)
+          @request = request
+          @session = session
+          @options = @request.options
+        end
+
+        def each(&block)
+          return enum_for(__method__) unless block_given?
+
+          raise Error, "response already streamed" if @response
+
+          @request.stream = self
+
+          begin
+            @on_chunk = block
+
+            response.raise_for_status
+            response.close
+          ensure
+            @on_chunk = nil
+          end
+        end
+
+        def each_line
+          return enum_for(__method__) unless block_given?
+
+          line = +""
+
+          each do |chunk|
+            line << chunk
+
+            while (idx = line.index("\n"))
+              yield line.byteslice(0..idx - 1)
+
+              line = line.byteslice(idx + 1..-1)
+            end
+          end
+        end
+
+        # This is a ghost method. It's to be used ONLY internally, when processing streams
+        def on_chunk(chunk)
+          raise NoMethodError unless @on_chunk
+
+          @on_chunk.call(chunk)
+        end
+
+        # :nocov:
+        def inspect
+          "#<StreamResponse:#{object_id}>"
+        end
+        # :nocov:
+
+        def to_s
+          response.to_s
+        end
+
+        private
+
+        def response
+          @response ||= @session.__send__(:send_requests, @request, @options).first
         end
 
-        def stream?
-          @headers["content-type"].start_with?("text/event-stream")
+        def respond_to_missing?(*args)
+          @options.response_class.respond_to?(*args) || super
         end
 
-        def <<(data)
-          res = super
-          @stream_complete = true if String(data).end_with?("\n\n")
-          res
+        def method_missing(meth, *args, &block)
+          if @options.response_class.public_method_defined?(meth)
+            response.__send__(meth, *args, &block)
+          else
+            super
+          end
         end
       end
     end