http 3.1.0 → 5.3.1

data/lib/http/options.rb

@@ -1,46 +1,45 @@
 # frozen_string_literal: true
 
-# rubocop:disable Metrics/ClassLength, Style/RedundantSelf
-
 require "http/headers"
 require "openssl"
 require "socket"
 require "http/uri"
-require "http/feature"
-require "http/features/auto_inflate"
-require "http/features/auto_deflate"
 
 module HTTP
-  class Options
+  class Options # rubocop:disable Metrics/ClassLength
     @default_socket_class     = TCPSocket
     @default_ssl_socket_class = OpenSSL::SSL::SSLSocket
     @default_timeout_class    = HTTP::Timeout::Null
-    @available_features       = {
-      :auto_inflate => Features::AutoInflate,
-      :auto_deflate => Features::AutoDeflate
-    }
+    @available_features       = {}
 
     class << self
       attr_accessor :default_socket_class, :default_ssl_socket_class, :default_timeout_class
       attr_reader :available_features
 
-      def new(options = {}) # rubocop:disable Style/OptionHash
-        return options if options.is_a?(self)
-        super
+      def new(options = {})
+        options.is_a?(self) ? options : super
       end
 
       def defined_options
         @defined_options ||= []
       end
 
+      def register_feature(name, impl)
+        @available_features[name] = impl
+      end
+
       protected
 
-      def def_option(name, &interpreter)
+      def def_option(name, reader_only: false, &interpreter)
         defined_options << name.to_sym
-        interpreter ||= lambda { |v| v }
+        interpreter ||= ->(v) { v }
 
-        attr_accessor name
-        protected :"#{name}="
+        if reader_only
+          attr_reader name
+        else
+          attr_accessor name
+          protected :"#{name}="
+        end
 
         define_method(:"with_#{name}") do |value|
           dup { |opts| opts.send(:"#{name}=", instance_exec(value, &interpreter)) }
@@ -48,7 +47,7 @@ module HTTP
       end
     end
 
-    def initialize(options = {}) # rubocop:disable Style/OptionHash
+    def initialize(options = {})
       defaults = {
         :response           => :auto,
         :proxy              => {},
@@ -70,12 +69,12 @@ module HTTP
       opts_w_defaults.each { |(k, v)| self[k] = v }
     end
 
-    def_option :headers do |headers|
-      self.headers.merge(headers)
+    def_option :headers do |new_headers|
+      headers.merge(new_headers)
     end
 
-    def_option :cookies do |cookies|
-      cookies.each_with_object self.cookies.dup do |(k, v), jar|
+    def_option :cookies do |new_cookies|
+      new_cookies.each_with_object cookies.dup do |(k, v), jar|
         cookie = k.is_a?(Cookie) ? k : Cookie.new(k.to_s, v.to_s)
         jar[cookie.name] = cookie.cookie_value
       end
@@ -85,7 +84,7 @@ module HTTP
       self.encoding = Encoding.find(encoding)
     end
 
-    def_option :features do |features|
+    def_option :features, :reader_only => true do |new_features|
       # Normalize features from:
       #
       #     [{feature_one: {opt: 'val'}}, :feature_two]
@@ -93,7 +92,7 @@ module HTTP
       # into:
       #
       #     {feature_one: {opt: 'val'}, feature_two: {}}
-      features = features.each_with_object({}) do |feature, h|
+      normalized_features = new_features.each_with_object({}) do |feature, h|
         if feature.is_a?(Hash)
           h.merge!(feature)
         else
@@ -101,7 +100,7 @@ module HTTP
         end
       end
 
-      self.features.merge(features)
+      features.merge(normalized_features)
     end
 
     def features=(features)
@@ -112,19 +111,21 @@ module HTTP
                     unless (feature = self.class.available_features[name])
                       argument_error! "Unsupported feature: #{name}"
                     end
-                    feature.new(opts_or_feature)
+                    feature.new(**opts_or_feature)
                   end
       end
     end
 
     %w[
-      proxy params form json body follow response
+      proxy params form json body response
       socket_class nodelay ssl_socket_class ssl_context ssl
-      persistent keep_alive_timeout timeout_class timeout_options
+      keep_alive_timeout timeout_class timeout_options
     ].each do |method_name|
       def_option method_name
     end
 
+    def_option :follow, :reader_only => true
+
     def follow=(value)
       @follow =
         case
@@ -135,6 +136,8 @@ module HTTP
         end
     end
 
+    def_option :persistent, :reader_only => true
+
     def persistent=(value)
       @persistent = value ? HTTP::URI.parse(value).origin : nil
     end

data/lib/http/redirector.rb

@@ -39,9 +39,10 @@ module HTTP
     # @param [Hash] opts
     # @option opts [Boolean] :strict (true) redirector hops policy
     # @option opts [#to_i] :max_hops (5) maximum allowed amount of hops
-    def initialize(opts = {}) # rubocop:disable Style/OptionHash
-      @strict   = opts.fetch(:strict, true)
-      @max_hops = opts.fetch(:max_hops, 5).to_i
+    def initialize(opts = {})
+      @strict      = opts.fetch(:strict, true)
+      @max_hops    = opts.fetch(:max_hops, 5).to_i
+      @on_redirect = opts.fetch(:on_redirect, nil)
     end
 
     # Follows redirects until non-redirect response found
@@ -49,6 +50,8 @@ module HTTP
       @request  = request
       @response = response
       @visited  = []
+      collect_cookies_from_request
+      collect_cookies_from_response
 
       while REDIRECT_CODES.include? @response.status.code
         @visited << "#{@request.verb} #{@request.uri}"
@@ -58,8 +61,14 @@ module HTTP
 
         @response.flush
 
-        @request  = redirect_to @response.headers[Headers::LOCATION]
+        # XXX(ixti): using `Array#inject` to return `nil` if no Location header.
+        @request = redirect_to(@response.headers.get(Headers::LOCATION).inject(:+))
+        unless cookie_jar.empty?
+          @request.headers.set(Headers::COOKIE, cookie_jar.cookies.map { |c| "#{c.name}=#{c.value}" }.join("; "))
+        end
+        @on_redirect.call @response, @request if @on_redirect.respond_to?(:call)
         @response = yield @request
+        collect_cookies_from_response
       end
 
       @response
@@ -67,6 +76,48 @@ module HTTP
 
     private
 
+    # All known cookies. On the original request, this is only the original cookies, but after that,
+    # Set-Cookie headers can add, set or delete cookies.
+    def cookie_jar
+      # it seems that @response.cookies instance is reused between responses, so we have to "clone"
+      @cookie_jar ||= HTTP::CookieJar.new
+    end
+
+    def collect_cookies_from_request
+      request_cookie_header = @request.headers["Cookie"]
+      cookies =
+        if request_cookie_header
+          HTTP::Cookie.cookie_value_to_hash(request_cookie_header)
+        else
+          {}
+        end
+
+      cookies.each do |key, value|
+        cookie_jar.add(HTTP::Cookie.new(key, value, :path => @request.uri.path, :domain => @request.host))
+      end
+    end
+
+    # Carry cookies from one response to the next. Carrying cookies to the next response ends up
+    # carrying them to the next request as well.
+    #
+    # Note that this isn't part of the IETF standard, but all major browsers support setting cookies
+    # on redirect: https://blog.dubbelboer.com/2012/11/25/302-cookie.html
+    def collect_cookies_from_response
+      # Overwrite previous cookies
+      @response.cookies.each do |cookie|
+        if cookie.value == ""
+          cookie_jar.delete(cookie)
+        else
+          cookie_jar.add(cookie)
+        end
+      end
+
+      # I wish we could just do @response.cookes = cookie_jar
+      cookie_jar.each do |cookie|
+        @response.cookies.add(cookie)
+      end
+    end
+
     # Check if we reached max amount of redirect hops
     # @return [Boolean]
     def too_many_hops?
@@ -89,6 +140,7 @@ module HTTP
 
       if UNSAFE_VERBS.include?(verb) && STRICT_SENSITIVE_CODES.include?(code)
         raise StateError, "can't follow #{@response.status} redirect" if @strict
+
         verb = :get
       end
 

data/lib/http/request/body.rb

@@ -19,6 +19,7 @@ module HTTP
           @source.bytesize
         elsif @source.respond_to?(:read)
           raise RequestError, "IO object must respond to #size" unless @source.respond_to?(:size)
+
           @source.size
         elsif @source.nil?
           0
@@ -35,13 +36,43 @@ module HTTP
           yield @source
         elsif @source.respond_to?(:read)
           IO.copy_stream(@source, ProcIO.new(block))
+          rewind(@source)
         elsif @source.is_a?(Enumerable)
           @source.each(&block)
         end
+
+        self
+      end
+
+      # Request bodies are equivalent when they have the same source.
+      def ==(other)
+        self.class == other.class && self.source == other.source # rubocop:disable Style/RedundantSelf
       end
 
       private
 
+      def rewind(io)
+        io.rewind if io.respond_to? :rewind
+      rescue Errno::ESPIPE, Errno::EPIPE
+        # Pipe IOs respond to `:rewind` but fail when you call it.
+        #
+        # Calling `IO#rewind` on a pipe, fails with *ESPIPE* on MRI,
+        # but *EPIPE* on jRuby.
+        #
+        # - **ESPIPE** -- "Illegal seek."
+        #   Invalid seek operation (such as on a pipe).
+        #
+        # - **EPIPE** -- "Broken pipe."
+        #   There is no process reading from the other end of a pipe. Every
+        #   library function that returns this error code also generates
+        #   a SIGPIPE signal; this signal terminates the program if not handled
+        #   or blocked. Thus, your program will never actually see EPIPE unless
+        #   it has handled or blocked SIGPIPE.
+        #
+        # See: https://www.gnu.org/software/libc/manual/html_node/Error-Codes.html
+        nil
+      end
+
       def validate_source_type!
         return if @source.is_a?(String)
         return if @source.respond_to?(:read)

data/lib/http/request/writer.rb

@@ -47,7 +47,11 @@ module HTTP
       # Adds the headers to the header array for the given request body we are working
       # with
       def add_body_type_headers
-        return if @headers[Headers::CONTENT_LENGTH] || chunked?
+        return if @headers[Headers::CONTENT_LENGTH] || chunked? || (
+          @body.source.nil? && %w[GET HEAD DELETE CONNECT].any? do |method|
+            @request_header[0].start_with?("#{method} ")
+          end
+        )
 
         @request_header << "#{Headers::CONTENT_LENGTH}: #{@body.size}"
       end
@@ -57,25 +61,35 @@ module HTTP
       def join_headers
         # join the headers array with crlfs, stick two on the end because
         # that ends the request header
-        @request_header.join(CRLF) + CRLF * 2
+        @request_header.join(CRLF) + (CRLF * 2)
       end
 
+      # Writes HTTP request data into the socket.
       def send_request
-        # It's important to send the request in a single write call when
-        # possible in order to play nicely with Nagle's algorithm. Making
-        # two writes in a row triggers a pathological case where Nagle is
-        # expecting a third write that never happens.
+        each_chunk { |chunk| write chunk }
+      rescue Errno::EPIPE
+        # server doesn't need any more data
+        nil
+      end
+
+      # Yields chunks of request data that should be sent to the socket.
+      #
+      # It's important to send the request in a single write call when possible
+      # in order to play nicely with Nagle's algorithm. Making two writes in a
+      # row triggers a pathological case where Nagle is expecting a third write
+      # that never happens.
+      def each_chunk
         data = join_headers
 
         @body.each do |chunk|
           data << encode_chunk(chunk)
-          write(data)
+          yield data
           data.clear
         end
 
-        write(data) unless data.empty?
+        yield data unless data.empty?
 
-        write(CHUNKED_END) if chunked?
+        yield CHUNKED_END if chunked?
       end
 
       # Returns the chunk encoded for to the specified "Transfer-Encoding" header.
@@ -94,12 +108,18 @@ module HTTP
 
       private
 
+      # @raise [SocketWriteError] when unable to write to socket
       def write(data)
         until data.empty?
           length = @socket.write(data)
           break unless data.bytesize > length
+
           data = data.byteslice(length..-1)
         end
+      rescue Errno::EPIPE
+        raise
+      rescue IOError, SocketError, SystemCallError => e
+        raise SocketWriteError, "error writing to socket: #{e}", e.backtrace
       end
     end
   end

data/lib/http/request.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 require "forwardable"
-require "base64"
 require "time"
 
+require "http/base64"
 require "http/errors"
 require "http/headers"
 require "http/request/body"
@@ -15,6 +15,7 @@ module HTTP
   class Request
     extend Forwardable
 
+    include HTTP::Base64
     include HTTP::Headers::Mixin
 
     # The method given was not understood
@@ -46,7 +47,13 @@ module HTTP
       :patch,
 
       # draft-reschke-webdav-search: WebDAV Search
-      :search
+      :search,
+
+      # RFC 4791: Calendaring Extensions to WebDAV -- CalDAV
+      :mkcalendar,
+
+      # Implemented by several caching servers, like Squid, Varnish or Fastly
+      :purge
     ].freeze
 
     # Allowed schemes
@@ -54,10 +61,10 @@ module HTTP
 
     # Default ports of supported schemes
     PORTS = {
-      :http   => 80,
-      :https  => 443,
-      :ws     => 80,
-      :wss    => 443
+      :http  => 80,
+      :https => 443,
+      :ws    => 80,
+      :wss   => 443
     }.freeze
 
     # Method is given as a lowercase symbol e.g. :get, :post
@@ -66,6 +73,8 @@ module HTTP
     # Scheme is normalized to be a lowercase symbol e.g. :http, :https
     attr_reader :scheme
 
+    attr_reader :uri_normalizer
+
     # "Request URI" as per RFC 2616
     # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html
     attr_reader :uri
@@ -73,25 +82,25 @@ module HTTP
 
     # @option opts [String] :version
     # @option opts [#to_s] :verb HTTP request method
+    # @option opts [#call] :uri_normalizer (HTTP::URI::NORMALIZER)
     # @option opts [HTTP::URI, #to_s] :uri
     # @option opts [Hash] :headers
     # @option opts [Hash] :proxy
     # @option opts [String, Enumerable, IO, nil] :body
     def initialize(opts)
-      @verb   = opts.fetch(:verb).to_s.downcase.to_sym
-      @uri    = normalize_uri(opts.fetch(:uri))
+      @verb           = opts.fetch(:verb).to_s.downcase.to_sym
+      @uri_normalizer = opts[:uri_normalizer] || HTTP::URI::NORMALIZER
+
+      @uri    = @uri_normalizer.call(opts.fetch(:uri))
       @scheme = @uri.scheme.to_s.downcase.to_sym if @uri.scheme
 
       raise(UnsupportedMethodError, "unknown method: #{verb}") unless METHODS.include?(@verb)
       raise(UnsupportedSchemeError, "unknown scheme: #{scheme}") unless SCHEMES.include?(@scheme)
 
       @proxy   = opts[:proxy] || {}
-      @body    = request_body(opts[:body], opts)
       @version = opts[:version] || "1.1"
-      @headers = HTTP::Headers.coerce(opts[:headers] || {})
-
-      @headers[Headers::HOST]        ||= default_host_header_value
-      @headers[Headers::USER_AGENT]  ||= USER_AGENT
+      @headers = prepare_headers(opts[:headers])
+      @body    = prepare_body(opts[:body])
     end
 
     # Returns new Request with updated uri
@@ -99,13 +108,28 @@ module HTTP
       headers = self.headers.dup
       headers.delete(Headers::HOST)
 
+      new_body = body.source
+      if verb == :get
+        # request bodies should not always be resubmitted when following a redirect
+        # some servers will close the connection after receiving the request headers
+        # which may cause Errno::ECONNRESET: Connection reset by peer
+        # see https://github.com/httprb/http/issues/649
+        # new_body = Request::Body.new(nil)
+        new_body = nil
+        # the CONTENT_TYPE header causes problems if set on a get request w/ an empty body
+        # the server might assume that there should be content if it is set to multipart
+        # rack raises EmptyContentError if this happens
+        headers.delete(Headers::CONTENT_TYPE)
+      end
+
       self.class.new(
-        :verb    => verb,
-        :uri     => @uri.join(uri),
-        :headers => headers,
-        :proxy   => proxy,
-        :body    => body,
-        :version => version
+        :verb           => verb,
+        :uri            => @uri.join(uri),
+        :headers        => headers,
+        :proxy          => proxy,
+        :body           => new_body,
+        :version        => version,
+        :uri_normalizer => uri_normalizer
       )
     end
 
@@ -136,7 +160,7 @@ module HTTP
     end
 
     def proxy_authorization_header
-      digest = Base64.strict_encode64("#{proxy[:proxy_username]}:#{proxy[:proxy_password]}")
+      digest = encode64("#{proxy[:proxy_username]}:#{proxy[:proxy_password]}")
       "Basic #{digest}"
     end
 
@@ -152,7 +176,9 @@ module HTTP
           uri.omit(:fragment)
         else
           uri.request_uri
-        end
+        end.to_s
+
+      raise RequestError, "Invalid request URI: #{request_uri.inspect}" if request_uri.match?(/\s/)
 
       "#{verb.to_s.upcase} #{request_uri} HTTP/#{version}"
     end
@@ -165,8 +191,8 @@ module HTTP
     # Headers to send with proxy connect request
     def proxy_connect_headers
       connect_headers = HTTP::Headers.coerce(
-        Headers::HOST        => headers[Headers::HOST],
-        Headers::USER_AGENT  => headers[Headers::USER_AGENT]
+        Headers::HOST       => headers[Headers::HOST],
+        Headers::USER_AGENT => headers[Headers::USER_AGENT]
       )
 
       connect_headers[Headers::PROXY_AUTHORIZATION] = proxy_authorization_header if using_authenticated_proxy?
@@ -184,15 +210,20 @@ module HTTP
       using_proxy? ? proxy[:proxy_port] : port
     end
 
-    private
-
-    # Transforms body to an object suitable for streaming.
-    def request_body(body, opts)
-      body = Request::Body.new(body) unless body.is_a?(Request::Body)
-      body = opts[:auto_deflate].deflated_body(body) if opts[:auto_deflate]
-      body
+    # Human-readable representation of base request info.
+    #
+    # @example
+    #
+    #     req.inspect
+    #     # => #<HTTP::Request/1.1 GET https://example.com>
+    #
+    # @return [String]
+    def inspect
+      "#<#{self.class}/#{@version} #{verb.to_s.upcase} #{uri}>"
     end
 
+    private
+
     # @!attribute [r] host
     #   @return [String]
     def_delegator :@uri, :host
@@ -205,20 +236,24 @@ module HTTP
 
     # @return [String] Default host (with port if needed) header value.
     def default_host_header_value
-      PORTS[@scheme] != port ? "#{host}:#{port}" : host
+      value = PORTS[@scheme] == port ? host : "#{host}:#{port}"
+
+      raise RequestError, "Invalid host: #{value.inspect}" if value.match?(/\s/)
+
+      value
     end
 
-    # @return [HTTP::URI] URI with all componentes but query being normalized.
-    def normalize_uri(uri)
-      uri = HTTP::URI.parse uri
+    def prepare_body(body)
+      body.is_a?(Request::Body) ? body : Request::Body.new(body)
+    end
 
-      HTTP::URI.new(
-        :scheme     => uri.normalized_scheme,
-        :authority  => uri.normalized_authority,
-        :path       => uri.normalized_path,
-        :query      => uri.query,
-        :fragment   => uri.normalized_fragment
-      )
+    def prepare_headers(headers)
+      headers = HTTP::Headers.coerce(headers || {})
+
+      headers[Headers::HOST]       ||= default_host_header_value
+      headers[Headers::USER_AGENT] ||= USER_AGENT
+
+      headers
     end
   end
 end

data/lib/http/response/body.rb

@@ -28,7 +28,8 @@ module HTTP
       def readpartial(*args)
         stream!
         chunk = @stream.readpartial(*args)
-        chunk.force_encoding(@encoding) if chunk
+
+        String.new(chunk, :encoding => @encoding) if chunk
       end
 
       # Iterate over the body, allowing it to be enumerable
@@ -46,11 +47,11 @@ module HTTP
 
         begin
           @streaming  = false
-          @contents   = String.new("").force_encoding(@encoding)
+          @contents   = String.new("", :encoding => @encoding)
 
           while (chunk = @stream.readpartial)
-            @contents << chunk.force_encoding(@encoding)
-            chunk.clear # deallocate string
+            @contents << String.new(chunk, :encoding => @encoding)
+            chunk = nil # deallocate string
           end
         rescue
           @contents = nil
@@ -64,6 +65,7 @@ module HTTP
       # Assert that the body is actively being streamed
       def stream!
         raise StateError, "body has already been consumed" if @streaming == false
+
         @streaming = true
       end
 

data/lib/http/response/inflater.rb

@@ -16,7 +16,7 @@ module HTTP
         if chunk
           chunk = zstream.inflate(chunk)
         elsif !zstream.closed?
-          zstream.finish
+          zstream.finish if zstream.total_in.positive?
           zstream.close
         end
         chunk