http 3.1.0 → 5.3.1

data/http.gemspec

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "http/version"
 
@@ -25,12 +25,21 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = HTTP::VERSION
 
-  gem.required_ruby_version = ">= 2.2"
+  gem.required_ruby_version = ">= 2.6"
 
-  gem.add_runtime_dependency "http_parser.rb", "~> 0.6.0"
-  gem.add_runtime_dependency "http-form_data", "~> 2.0"
+  gem.add_runtime_dependency "addressable",    "~> 2.8"
   gem.add_runtime_dependency "http-cookie",    "~> 1.0"
-  gem.add_runtime_dependency "addressable",    "~> 2.3"
+  gem.add_runtime_dependency "http-form_data", "~> 2.2"
 
-  gem.add_development_dependency "bundler", "~> 1.0"
+  gem.add_runtime_dependency "llhttp-ffi", "~> 0.5.0"
+
+  gem.add_development_dependency "bundler", "~> 2.0"
+
+  gem.metadata = {
+    "source_code_uri"       => "https://github.com/httprb/http",
+    "wiki_uri"              => "https://github.com/httprb/http/wiki",
+    "bug_tracker_uri"       => "https://github.com/httprb/http/issues",
+    "changelog_uri"         => "https://github.com/httprb/http/blob/v#{HTTP::VERSION}/CHANGELOG.md",
+    "rubygems_mfa_required" => "true"
+  }
 end

data/lib/http/base64.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module HTTP
+  module Base64
+    module_function
+
+    # Equivalent to Base64.strict_encode64
+    def encode64(input)
+      [input].pack("m0")
+    end
+  end
+end

data/lib/http/chainable.rb

@@ -1,111 +1,113 @@
 # frozen_string_literal: true
 
-require "base64"
-
+require "http/base64"
 require "http/headers"
 
 module HTTP
   module Chainable
+    include HTTP::Base64
+
     # Request a get sans response body
     # @param uri
     # @option options [Hash]
-    def head(uri, options = {}) # rubocop:disable Style/OptionHash
+    def head(uri, options = {})
       request :head, uri, options
     end
 
     # Get a resource
     # @param uri
     # @option options [Hash]
-    def get(uri, options = {}) # rubocop:disable Style/OptionHash
+    def get(uri, options = {})
       request :get, uri, options
     end
 
     # Post to a resource
     # @param uri
     # @option options [Hash]
-    def post(uri, options = {}) # rubocop:disable Style/OptionHash
+    def post(uri, options = {})
       request :post, uri, options
     end
 
     # Put to a resource
     # @param uri
     # @option options [Hash]
-    def put(uri, options = {}) # rubocop:disable Style/OptionHash
+    def put(uri, options = {})
       request :put, uri, options
     end
 
     # Delete a resource
     # @param uri
     # @option options [Hash]
-    def delete(uri, options = {}) # rubocop:disable Style/OptionHash
+    def delete(uri, options = {})
       request :delete, uri, options
     end
 
     # Echo the request back to the client
     # @param uri
     # @option options [Hash]
-    def trace(uri, options = {}) # rubocop:disable Style/OptionHash
+    def trace(uri, options = {})
       request :trace, uri, options
     end
 
     # Return the methods supported on the given URI
     # @param uri
     # @option options [Hash]
-    def options(uri, options = {}) # rubocop:disable Style/OptionHash
+    def options(uri, options = {})
       request :options, uri, options
     end
 
     # Convert to a transparent TCP/IP tunnel
     # @param uri
     # @option options [Hash]
-    def connect(uri, options = {}) # rubocop:disable Style/OptionHash
+    def connect(uri, options = {})
       request :connect, uri, options
     end
 
     # Apply partial modifications to a resource
     # @param uri
     # @option options [Hash]
-    def patch(uri, options = {}) # rubocop:disable Style/OptionHash
+    def patch(uri, options = {})
       request :patch, uri, options
     end
 
     # Make an HTTP request with the given verb
-    # @param uri
-    # @option options [Hash]
-    def request(verb, uri, options = {}) # rubocop:disable Style/OptionHash
-      branch(options).request verb, uri
+    # @param (see Client#request)
+    def request(*args)
+      branch(default_options).request(*args)
+    end
+
+    # Prepare an HTTP request with the given verb
+    # @param (see Client#build_request)
+    def build_request(*args)
+      branch(default_options).build_request(*args)
     end
 
     # @overload timeout(options = {})
-    #   Syntax sugar for `timeout(:per_operation, options)`
-    # @overload timeout(klass, options = {})
-    #   Adds a timeout to the request.
-    #   @param [#to_sym] klass
-    #     either :null, :global, or :per_operation
+    #   Adds per operation timeouts to the request
     #   @param [Hash] options
     #   @option options [Float] :read Read timeout
     #   @option options [Float] :write Write timeout
     #   @option options [Float] :connect Connect timeout
-    def timeout(klass, options = {}) # rubocop:disable Style/OptionHash
-      if klass.is_a? Hash
-        options = klass
-        klass   = :per_operation
-      end
-
-      klass = case klass.to_sym
-              when :null          then HTTP::Timeout::Null
-              when :global        then HTTP::Timeout::Global
-              when :per_operation then HTTP::Timeout::PerOperation
-              else raise ArgumentError, "Unsupported Timeout class: #{klass}"
-              end
-
-      %i[read write connect].each do |k|
+    # @overload timeout(global_timeout)
+    #   Adds a global timeout to the full request
+    #   @param [Numeric] global_timeout
+    def timeout(options)
+      klass, options = case options
+                       when Numeric then [HTTP::Timeout::Global, {:global => options}]
+                       when Hash    then [HTTP::Timeout::PerOperation, options.dup]
+                       when :null   then [HTTP::Timeout::Null, {}]
+                       else raise ArgumentError, "Use `.timeout(global_timeout_in_seconds)` or `.timeout(connect: x, write: y, read: z)`."
+
+                       end
+
+      %i[global read write connect].each do |k|
         next unless options.key? k
+
         options["#{k}_timeout".to_sym] = options.delete k
       end
 
       branch default_options.merge(
-        :timeout_class => klass,
+        :timeout_class   => klass,
         :timeout_options => options
       )
     end
@@ -144,9 +146,10 @@ module HTTP
       options  = {:keep_alive_timeout => timeout}
       p_client = branch default_options.merge(options).with_persistent host
       return p_client unless block_given?
+
       yield p_client
     ensure
-      p_client.close if p_client
+      p_client&.close
     end
 
     # Make a request through an HTTP proxy
@@ -168,10 +171,10 @@ module HTTP
     alias through via
 
     # Make client follow redirects.
-    # @param opts
+    # @param options
     # @return [HTTP::Client]
     # @see Redirector#initialize
-    def follow(options = {}) # rubocop:disable Style/OptionHash
+    def follow(options = {})
       branch default_options.with_follow options
     end
 
@@ -209,10 +212,11 @@ module HTTP
     # @option opts [#to_s] :user
     # @option opts [#to_s] :pass
     def basic_auth(opts)
-      user = opts.fetch :user
-      pass = opts.fetch :pass
+      user  = opts.fetch(:user)
+      pass  = opts.fetch(:pass)
+      creds = "#{user}:#{pass}"
 
-      auth("Basic " + Base64.strict_encode64("#{user}:#{pass}"))
+      auth("Basic #{encode64(creds)}")
     end
 
     # Get options for HTTP
@@ -236,11 +240,37 @@ module HTTP
     # Turn on given features. Available features are:
     # * auto_inflate
     # * auto_deflate
+    # * instrumentation
+    # * logging
+    # * normalize_uri
+    # * raise_error
     # @param features
     def use(*features)
       branch default_options.with_features(features)
     end
 
+    # Returns retriable client instance, which retries requests if they failed
+    # due to some socket errors or response status is `5xx`.
+    #
+    # @example Usage
+    #
+    #   # Retry max 5 times with randomly growing delay between retries
+    #   HTTP.retriable.get(url)
+    #
+    #   # Retry max 3 times with randomly growing delay between retries
+    #   HTTP.retriable(times: 3).get(url)
+    #
+    #   # Retry max 3 times with 1 sec delay between retries
+    #   HTTP.retriable(times: 3, delay: proc { 1 }).get(url)
+    #
+    #   # Retry max 3 times with geometrically progressed delay between retries
+    #   HTTP.retriable(times: 3, delay: proc { |i| 1 + i*i }).get(url)
+    #
+    # @param (see Performer#initialize)
+    def retriable(**options)
+      Retriable::Client.new(Retriable::Performer.new(options), default_options)
+    end
+
     private
 
     # :nodoc:

data/lib/http/client.rb

@@ -4,6 +4,7 @@ require "forwardable"
 
 require "http/form_data"
 require "http/options"
+require "http/feature"
 require "http/headers"
 require "http/connection"
 require "http/redirector"
@@ -15,7 +16,7 @@ module HTTP
     extend Forwardable
     include Chainable
 
-    HTTP_OR_HTTPS_RE = %r{^https?://}i
+    HTTP_OR_HTTPS_RE = %r{^https?://}i.freeze
 
     def initialize(default_options = {})
       @default_options = HTTP::Options.new(default_options)
@@ -24,28 +25,34 @@ module HTTP
     end
 
     # Make an HTTP request
-    def request(verb, uri, opts = {}) # rubocop:disable Style/OptionHash
+    def request(verb, uri, opts = {})
+      opts = @default_options.merge(opts)
+      req = build_request(verb, uri, opts)
+      res = perform(req, opts)
+      return res unless opts.follow
+
+      Redirector.new(opts.follow).perform(req, res) do |request|
+        perform(wrap_request(request, opts), opts)
+      end
+    end
+
+    # Prepare an HTTP request
+    def build_request(verb, uri, opts = {})
       opts    = @default_options.merge(opts)
       uri     = make_request_uri(uri, opts)
       headers = make_request_headers(opts)
       body    = make_request_body(opts, headers)
-      proxy   = opts.proxy
 
       req = HTTP::Request.new(
-        :verb         => verb,
-        :uri          => uri,
-        :headers      => headers,
-        :proxy        => proxy,
-        :body         => body,
-        :auto_deflate => opts.feature(:auto_deflate)
+        :verb           => verb,
+        :uri            => uri,
+        :uri_normalizer => opts.feature(:normalize_uri)&.normalizer,
+        :proxy          => opts.proxy,
+        :headers        => headers,
+        :body           => body
       )
 
-      res = perform(req, opts)
-      return res unless opts.follow
-
-      Redirector.new(opts.follow).perform(req, res) do |request|
-        perform(request, opts)
-      end
+      wrap_request(req, opts)
     end
 
     # @!method persistent?
@@ -59,23 +66,24 @@ module HTTP
 
       @state = :dirty
 
-      @connection ||= HTTP::Connection.new(req, options)
-
-      unless @connection.failed_proxy_connect?
-        @connection.send_request(req)
-        @connection.read_headers!
+      begin
+        @connection ||= HTTP::Connection.new(req, options)
+
+        unless @connection.failed_proxy_connect?
+          @connection.send_request(req)
+          @connection.read_headers!
+        end
+      rescue Error => e
+        options.features.each_value do |feature|
+          feature.on_error(req, e)
+        end
+        raise
       end
+      res = build_response(req, options)
 
-      res = Response.new(
-        :status        => @connection.status_code,
-        :version       => @connection.http_version,
-        :headers       => @connection.headers,
-        :proxy_headers => @connection.proxy_response_headers,
-        :connection    => @connection,
-        :encoding      => options.encoding,
-        :auto_inflate  => options.feature(:auto_inflate),
-        :uri           => req.uri
-      )
+      res = options.features.inject(res) do |response, (_name, feature)|
+        feature.wrap_response(response)
+      end
 
       @connection.finish_response if req.verb == :head
       @state = :clean
@@ -87,26 +95,44 @@ module HTTP
     end
 
     def close
-      @connection.close if @connection
+      @connection&.close
       @connection = nil
       @state = :clean
     end
 
     private
 
+    def wrap_request(req, opts)
+      opts.features.inject(req) do |request, (_name, feature)|
+        feature.wrap_request(request)
+      end
+    end
+
+    def build_response(req, options)
+      Response.new(
+        :status        => @connection.status_code,
+        :version       => @connection.http_version,
+        :headers       => @connection.headers,
+        :proxy_headers => @connection.proxy_response_headers,
+        :connection    => @connection,
+        :encoding      => options.encoding,
+        :request       => req
+      )
+    end
+
     # Verify our request isn't going to be made against another URI
     def verify_connection!(uri)
       if default_options.persistent? && uri.origin != default_options.persistent
         raise StateError, "Persistence is enabled for #{default_options.persistent}, but we got #{uri.origin}"
+      end
+
       # We re-create the connection object because we want to let prior requests
       # lazily load the body as long as possible, and this mimics prior functionality.
-      elsif @connection && (!@connection.keep_alive? || @connection.expired?)
-        close
+      return close if @connection && (!@connection.keep_alive? || @connection.expired?)
+
       # If we get into a bad state (eg, Timeout.timeout ensure being killed)
       # close the connection to prevent potential for mixed responses.
-      elsif @state == :dirty
-        close
-      end
+      return close if @state == :dirty
     end
 
     # Merges query params if needed
@@ -116,15 +142,11 @@ module HTTP
     def make_request_uri(uri, opts)
       uri = uri.to_s
 
-      if default_options.persistent? && uri !~ HTTP_OR_HTTPS_RE
-        uri = "#{default_options.persistent}#{uri}"
-      end
+      uri = "#{default_options.persistent}#{uri}" if default_options.persistent? && uri !~ HTTP_OR_HTTPS_RE
 
       uri = HTTP::URI.parse uri
 
-      if opts.params && !opts.params.empty?
-        uri.query_values = uri.query_values(Array).to_a.concat(opts.params.to_a)
-      end
+      uri.query_values = uri.query_values(Array).to_a.concat(opts.params.to_a) if opts.params && !opts.params.empty?
 
       # Some proxies (seen on WEBRick) fail if URL has
       # empty path (e.g. `http://example.com`) while it's RFC-complaint:
@@ -148,14 +170,6 @@ module HTTP
         headers[Headers::COOKIE] = cookies
       end
 
-      if (auto_deflate = opts.feature(:auto_deflate))
-        # We need to delete Content-Length header. It will be set automatically
-        # by HTTP::Request::Writer
-        headers.delete(Headers::CONTENT_LENGTH)
-
-        headers[Headers::CONTENT_ENCODING] = auto_deflate.method
-      end
-
       headers
     end
 
@@ -165,14 +179,21 @@ module HTTP
       when opts.body
         opts.body
       when opts.form
-        form = HTTP::FormData.create opts.form
+        form = make_form_data(opts.form)
         headers[Headers::CONTENT_TYPE] ||= form.content_type
         form
       when opts.json
         body = MimeType[:json].encode opts.json
-        headers[Headers::CONTENT_TYPE] ||= "application/json; charset=#{body.encoding.name}"
+        headers[Headers::CONTENT_TYPE] ||= "application/json; charset=#{body.encoding.name.downcase}"
         body
       end
     end
+
+    def make_form_data(form)
+      return form if form.is_a? HTTP::FormData::Multipart
+      return form if form.is_a? HTTP::FormData::Urlencoded
+
+      HTTP::FormData.create(form)
+    end
   end
 end

data/lib/http/connection.rb

@@ -3,11 +3,10 @@
 require "forwardable"
 
 require "http/headers"
-require "http/response/parser"
 
 module HTTP
   # A connection to the HTTP server
-  class Connection # rubocop: disable Metrics/ClassLength
+  class Connection
     extend Forwardable
 
     # Allowed values for CONNECTION header
@@ -35,6 +34,7 @@ module HTTP
       @pending_request      = false
       @pending_response     = false
       @failed_proxy_connect = false
+      @buffer               = "".b
 
       @parser = Response::Parser.new
 
@@ -44,8 +44,11 @@ module HTTP
       send_proxy_connect_request(req)
       start_tls(req, options)
       reset_timer
-    rescue IOError, SocketError, SystemCallError => ex
-      raise ConnectionError, "failed to connect: #{ex}", ex.backtrace
+    rescue IOError, SocketError, SystemCallError => e
+      raise ConnectionError, "failed to connect: #{e}", e.backtrace
+    rescue TimeoutError
+      close
+      raise
     end
 
     # @see (HTTP::Response::Parser#status_code)
@@ -67,8 +70,13 @@ module HTTP
     # @param [Request] req Request to send to the server
     # @return [nil]
     def send_request(req)
-      raise StateError, "Tried to send a request while one is pending already. Make sure you read off the body." if @pending_response
-      raise StateError, "Tried to send a request while a response is pending. Make sure you read off the body."  if @pending_request
+      if @pending_response
+        raise StateError, "Tried to send a request while one is pending already. Make sure you read off the body."
+      end
+
+      if @pending_request
+        raise StateError, "Tried to send a request while a response is pending. Make sure you read off the body."
+      end
 
       @pending_request = true
 
@@ -92,19 +100,16 @@ module HTTP
       chunk    = @parser.read(size)
       finish_response if finished
 
-      chunk.to_s
+      chunk || "".b
     end
 
     # Reads data from socket up until headers are loaded
     # @return [void]
+    # @raise [ResponseHeaderError] when unable to read response headers
     def read_headers!
-      loop do
-        if read_more(BUFFER_SIZE) == :eof
-          raise ConnectionError, "couldn't read response headers" unless @parser.headers?
-          break
-        elsif @parser.headers?
-          break
-        end
+      until @parser.headers?
+        result = read_more(BUFFER_SIZE)
+        raise ResponseHeaderError, "couldn't read response headers" if result == :eof
       end
 
       set_keep_alive
@@ -125,12 +130,16 @@ module HTTP
     # Close the connection
     # @return [void]
     def close
-      @socket.close unless @socket.closed?
+      @socket.close unless @socket&.closed?
 
       @pending_response = false
       @pending_request  = false
     end
 
+    def finished_request?
+      !@pending_request && !@pending_response
+    end
+
     # Whether we're keeping the conn alive
     # @return [Boolean]
     def keep_alive?
@@ -209,18 +218,19 @@ module HTTP
 
     # Feeds some more data into parser
     # @return [void]
+    # @raise [SocketReadError] when unable to read from socket
     def read_more(size)
       return if @parser.finished?
 
-      value = @socket.readpartial(size)
+      value = @socket.readpartial(size, @buffer)
       if value == :eof
         @parser << ""
         :eof
       elsif value
         @parser << value
       end
-    rescue IOError, SocketError, SystemCallError => ex
-      raise ConnectionError, "error reading from socket: #{ex}", ex.backtrace
+    rescue IOError, SocketError, SystemCallError => e
+      raise SocketReadError, "error reading from socket: #{e}", e.backtrace
     end
   end
 end

data/lib/http/content_type.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
 module HTTP
-  ContentType = Struct.new(:mime_type, :charset) do
-    MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}
-    CHARSET_RE   = /;\s*charset=([^;]+)/i
+  class ContentType
+    MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
+    CHARSET_RE   = /;\s*charset=([^;]+)/i.freeze
+
+    attr_accessor :mime_type, :charset
 
     class << self
       # Parse string and return ContentType struct
@@ -15,15 +17,18 @@ module HTTP
 
       # :nodoc:
       def mime_type(str)
-        m = str.to_s[MIME_TYPE_RE, 1]
-        m && m.strip.downcase
+        str.to_s[MIME_TYPE_RE, 1]&.strip&.downcase
       end
 
       # :nodoc:
       def charset(str)
-        m = str.to_s[CHARSET_RE, 1]
-        m && m.strip.delete('"')
+        str.to_s[CHARSET_RE, 1]&.strip&.delete('"')
       end
     end
+
+    def initialize(mime_type = nil, charset = nil)
+      @mime_type = mime_type
+      @charset   = charset
+    end
   end
 end

data/lib/http/errors.rb

@@ -7,6 +7,11 @@ module HTTP
   # Generic Connection error
   class ConnectionError < Error; end
 
+  # Types of Connection errors
+  class ResponseHeaderError < ConnectionError; end
+  class SocketReadError < ConnectionError; end
+  class SocketWriteError < ConnectionError; end
+
   # Generic Request error
   class RequestError < Error; end
 
@@ -16,9 +21,23 @@ module HTTP
   # Requested to do something when we're in the wrong state
   class StateError < ResponseError; end
 
+  # When status code indicates an error
+  class StatusError < ResponseError
+    attr_reader :response
+
+    def initialize(response)
+      @response = response
+
+      super("Unexpected status code #{response.code}")
+    end
+  end
+
   # Generic Timeout error
   class TimeoutError < Error; end
 
+  # Timeout when first establishing the conncetion
+  class ConnectTimeoutError < TimeoutError; end
+
   # Header value is of unexpected format (similar to Net::HTTPHeaderSyntaxError)
   class HeaderError < Error; end
 end