http 5.1.1 → 5.3.0

data/lib/http/headers.rb

@@ -4,6 +4,7 @@ require "forwardable"
 
 require "http/errors"
 require "http/headers/mixin"
+require "http/headers/normalizer"
 require "http/headers/known"
 
 module HTTP
@@ -12,12 +13,31 @@ module HTTP
     extend Forwardable
     include Enumerable
 
-    # Matches HTTP header names when in "Canonical-Http-Format"
-    CANONICAL_NAME_RE = /\A[A-Z][a-z]*(?:-[A-Z][a-z]*)*\z/.freeze
+    class << self
+      # Coerces given `object` into Headers.
+      #
+      # @raise [Error] if object can't be coerced
+      # @param [#to_hash, #to_h, #to_a] object
+      # @return [Headers]
+      def coerce(object)
+        unless object.is_a? self
+          object = case
+                   when object.respond_to?(:to_hash) then object.to_hash
+                   when object.respond_to?(:to_h)    then object.to_h
+                   when object.respond_to?(:to_a)    then object.to_a
+                   else raise Error, "Can't coerce #{object.inspect} to Headers"
+                   end
+        end
+        headers = new
+        object.each { |k, v| headers.add k, v }
+        headers
+      end
+      alias [] coerce
 
-    # Matches valid header field name according to RFC.
-    # @see http://tools.ietf.org/html/rfc7230#section-3.2
-    COMPLIANT_NAME_RE = /\A[A-Za-z0-9!#$%&'*+\-.^_`|~]+\z/.freeze
+      def normalizer
+        @normalizer ||= Headers::Normalizer.new
+      end
+    end
 
     # Class constructor.
     def initialize
@@ -194,45 +214,11 @@ module HTTP
       dup.tap { |dupped| dupped.merge! other }
     end
 
-    class << self
-      # Coerces given `object` into Headers.
-      #
-      # @raise [Error] if object can't be coerced
-      # @param [#to_hash, #to_h, #to_a] object
-      # @return [Headers]
-      def coerce(object)
-        unless object.is_a? self
-          object = case
-                   when object.respond_to?(:to_hash) then object.to_hash
-                   when object.respond_to?(:to_h)    then object.to_h
-                   when object.respond_to?(:to_a)    then object.to_a
-                   else raise Error, "Can't coerce #{object.inspect} to Headers"
-                   end
-        end
-
-        headers = new
-        object.each { |k, v| headers.add k, v }
-        headers
-      end
-      alias [] coerce
-    end
-
     private
 
     # Transforms `name` to canonical HTTP header capitalization
-    #
-    # @param [String] name
-    # @raise [HeaderError] if normalized name does not
-    #   match {HEADER_NAME_RE}
-    # @return [String] canonical HTTP header name
     def normalize_header(name)
-      return name if name =~ CANONICAL_NAME_RE
-
-      normalized = name.split(/[\-_]/).each(&:capitalize!).join("-")
-
-      return normalized if normalized =~ COMPLIANT_NAME_RE
-
-      raise HeaderError, "Invalid HTTP header field name: #{name.inspect}"
+      self.class.normalizer.call(name)
     end
 
     # Ensures there is no new line character in the header value

data/lib/http/request/writer.rb

@@ -108,6 +108,7 @@ module HTTP
 
       private
 
+      # @raise [SocketWriteError] when unable to write to socket
       def write(data)
         until data.empty?
           length = @socket.write(data)
@@ -118,7 +119,7 @@ module HTTP
       rescue Errno::EPIPE
         raise
       rescue IOError, SocketError, SystemCallError => e
-        raise ConnectionError, "error writing to socket: #{e}", e.backtrace
+        raise SocketWriteError, "error writing to socket: #{e}", e.backtrace
       end
     end
   end

data/lib/http/request.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 require "forwardable"
-require "base64"
 require "time"
 
+require "http/base64"
 require "http/errors"
 require "http/headers"
 require "http/request/body"
@@ -15,6 +15,7 @@ module HTTP
   class Request
     extend Forwardable
 
+    include HTTP::Base64
     include HTTP::Headers::Mixin
 
     # The method given was not understood
@@ -49,7 +50,10 @@ module HTTP
       :search,
 
       # RFC 4791: Calendaring Extensions to WebDAV -- CalDAV
-      :mkcalendar
+      :mkcalendar,
+
+      # Implemented by several caching servers, like Squid, Varnish or Fastly
+      :purge
     ].freeze
 
     # Allowed schemes
@@ -156,7 +160,7 @@ module HTTP
     end
 
     def proxy_authorization_header
-      digest = Base64.strict_encode64("#{proxy[:proxy_username]}:#{proxy[:proxy_password]}")
+      digest = encode64("#{proxy[:proxy_username]}:#{proxy[:proxy_password]}")
       "Basic #{digest}"
     end
 
@@ -172,7 +176,9 @@ module HTTP
           uri.omit(:fragment)
         else
           uri.request_uri
-        end
+        end.to_s
+
+      raise RequestError, "Invalid request URI: #{request_uri.inspect}" if request_uri.match?(/\s/)
 
       "#{verb.to_s.upcase} #{request_uri} HTTP/#{version}"
     end
@@ -230,7 +236,11 @@ module HTTP
 
     # @return [String] Default host (with port if needed) header value.
     def default_host_header_value
-      PORTS[@scheme] == port ? host : "#{host}:#{port}"
+      value = PORTS[@scheme] == port ? host : "#{host}:#{port}"
+
+      raise RequestError, "Invalid host: #{value.inspect}" if value.match?(/\s/)
+
+      value
     end
 
     def prepare_body(body)

data/lib/http/retriable/client.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "http/retriable/performer"
+
+module HTTP
+  module Retriable
+    # Retriable version of HTTP::Client.
+    #
+    # @see http://www.rubydoc.info/gems/http/HTTP/Client
+    class Client < HTTP::Client
+      # @param [Performer] performer
+      # @param [HTTP::Options, Hash] options
+      def initialize(performer, options)
+        @performer = performer
+        super(options)
+      end
+
+      # Overriden version of `HTTP::Client#make_request`.
+      #
+      # Monitors request/response phase with performer.
+      #
+      # @see http://www.rubydoc.info/gems/http/HTTP/Client:perform
+      def perform(req, options)
+        @performer.perform(self, req) { super(req, options) }
+      end
+
+      private
+
+      # Overriden version of `HTTP::Chainable#branch`.
+      #
+      # @return [HTTP::Retriable::Client]
+      def branch(options)
+        Retriable::Client.new(@performer, options)
+      end
+    end
+  end
+end

data/lib/http/retriable/delay_calculator.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module HTTP
+  module Retriable
+    # @api private
+    class DelayCalculator
+      def initialize(opts)
+        @max_delay = opts.fetch(:max_delay, Float::MAX).to_f
+        if (delay = opts[:delay]).respond_to?(:call)
+          @delay_proc = opts.fetch(:delay)
+        else
+          @delay = delay
+        end
+      end
+
+      def call(iteration, response)
+        delay = if response && (retry_header = response.headers["Retry-After"])
+                  delay_from_retry_header(retry_header)
+                else
+                  calculate_delay_from_iteration(iteration)
+                end
+
+        ensure_dealy_in_bounds(delay)
+      end
+
+      RFC2822_DATE_REGEX = /^
+        (?:Sun|Mon|Tue|Wed|Thu|Fri|Sat),\s+
+        (?:0[1-9]|[1-2]?[0-9]|3[01])\s+
+        (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+
+        (?:19[0-9]{2}|[2-9][0-9]{3})\s+
+        (?:2[0-3]|[0-1][0-9]):(?:[0-5][0-9]):(?:60|[0-5][0-9])\s+
+        GMT
+      $/x
+
+      # Spec for Retry-After header
+      # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+      def delay_from_retry_header(value)
+        value = value.to_s.strip
+
+        case value
+        when RFC2822_DATE_REGEX then DateTime.rfc2822(value).to_time - Time.now.utc
+        when /^\d+$/            then value.to_i
+        else 0
+        end
+      end
+
+      def calculate_delay_from_iteration(iteration)
+        if @delay_proc
+          @delay_proc.call(iteration)
+        elsif @delay
+          @delay
+        else
+          delay = (2**(iteration - 1)) - 1
+          delay_noise = rand
+          delay + delay_noise
+        end
+      end
+
+      def ensure_dealy_in_bounds(delay)
+        delay.clamp(0, @max_delay)
+      end
+    end
+  end
+end

data/lib/http/retriable/errors.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module HTTP
+  # Retriable performance ran out of attempts
+  class OutOfRetriesError < Error
+    attr_accessor :response
+
+    attr_writer :cause
+
+    def cause
+      @cause || super
+    end
+  end
+end

data/lib/http/retriable/performer.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require "date"
+require "http/retriable/errors"
+require "http/retriable/delay_calculator"
+require "openssl"
+
+module HTTP
+  module Retriable
+    # Request performing watchdog.
+    # @api private
+    class Performer
+      # Exceptions we should retry
+      RETRIABLE_ERRORS = [
+        HTTP::TimeoutError,
+        HTTP::ConnectionError,
+        IO::EAGAINWaitReadable,
+        Errno::ECONNRESET,
+        Errno::ECONNREFUSED,
+        Errno::EHOSTUNREACH,
+        OpenSSL::SSL::SSLError,
+        EOFError,
+        IOError
+      ].freeze
+
+      # @param [Hash] opts
+      # @option opts [#to_i] :tries (5)
+      # @option opts [#call, #to_i] :delay (DELAY_PROC)
+      # @option opts [Array(Exception)] :exceptions (RETRIABLE_ERRORS)
+      # @option opts [Array(#to_i)] :retry_statuses
+      # @option opts [#call] :on_retry
+      # @option opts [#to_f] :max_delay (Float::MAX)
+      # @option opts [#call] :should_retry
+      def initialize(opts)
+        @exception_classes = opts.fetch(:exceptions, RETRIABLE_ERRORS)
+        @retry_statuses = opts[:retry_statuses]
+        @tries = opts.fetch(:tries, 5).to_i
+        @on_retry = opts.fetch(:on_retry, ->(*) {})
+        @should_retry_proc = opts[:should_retry]
+        @delay_calculator = DelayCalculator.new(opts)
+      end
+
+      # Watches request/response execution.
+      #
+      # If any of {RETRIABLE_ERRORS} occur or response status is `5xx`, retries
+      # up to `:tries` amount of times. Sleeps for amount of seconds calculated
+      # with `:delay` proc before each retry.
+      #
+      # @see #initialize
+      # @api private
+      def perform(client, req, &block)
+        1.upto(Float::INFINITY) do |attempt| # infinite loop with index
+          err, res = try_request(&block)
+
+          if retry_request?(req, err, res, attempt)
+            begin
+              wait_for_retry_or_raise(req, err, res, attempt)
+            ensure
+              # Some servers support Keep-Alive on any response. Thus we should
+              # flush response before retry, to avoid state error (when socket
+              # has pending response data and we try to write new request).
+              # Alternatively, as we don't need response body here at all, we
+              # are going to close client, effectivle closing underlying socket
+              # and resetting client's state.
+              client.close
+            end
+          elsif err
+            client.close
+            raise err
+          elsif res
+            return res
+          end
+        end
+      end
+
+      def calculate_delay(iteration, response)
+        @delay_calculator.call(iteration, response)
+      end
+
+      private
+
+      # rubocop:disable Lint/RescueException
+      def try_request
+        err, res = nil
+
+        begin
+          res = yield
+        rescue Exception => e
+          err = e
+        end
+
+        [err, res]
+      end
+      # rubocop:enable Lint/RescueException
+
+      def retry_request?(req, err, res, attempt)
+        if @should_retry_proc
+          @should_retry_proc.call(req, err, res, attempt)
+        elsif err
+          retry_exception?(err)
+        else
+          retry_response?(res)
+        end
+      end
+
+      def retry_exception?(err)
+        @exception_classes.any? { |e| err.is_a?(e) }
+      end
+
+      def retry_response?(res)
+        return false unless @retry_statuses
+
+        response_status = res.status.to_i
+        retry_matchers = [@retry_statuses].flatten
+
+        retry_matchers.any? do |matcher|
+          case matcher
+          when Range then matcher.cover?(response_status)
+          when Numeric then matcher == response_status
+          else matcher.call(response_status)
+          end
+        end
+      end
+
+      def wait_for_retry_or_raise(req, err, res, attempt)
+        if attempt < @tries
+          @on_retry.call(req, err, res)
+          sleep calculate_delay(attempt, res)
+        else
+          res&.flush
+          raise out_of_retries_error(req, res, err)
+        end
+      end
+
+      # Builds OutOfRetriesError
+      #
+      # @param request [HTTP::Request]
+      # @param status [HTTP::Response, nil]
+      # @param exception [Exception, nil]
+      def out_of_retries_error(request, response, exception)
+        message = "#{request.verb.to_s.upcase} <#{request.uri}> failed"
+
+        message += " with #{response.status}" if response
+        message += ":#{exception}" if exception
+
+        HTTP::OutOfRetriesError.new(message).tap do |ex|
+          ex.cause = exception
+          ex.response = response
+        end
+      end
+    end
+  end
+end

data/lib/http/timeout/null.rb

@@ -1,15 +1,10 @@
 # frozen_string_literal: true
 
-require "forwardable"
 require "io/wait"
 
 module HTTP
   module Timeout
     class Null
-      extend Forwardable
-
-      def_delegators :@socket, :close, :closed?
-
       attr_reader :options, :socket
 
       def initialize(options = {})
@@ -27,6 +22,14 @@ module HTTP
         @socket.connect
       end
 
+      def close
+        @socket&.close
+      end
+
+      def closed?
+        @socket&.closed?
+      end
+
       # Configures the SSL connection and starts the connection
       def start_tls(host, ssl_socket_class, ssl_context)
         @socket = ssl_socket_class.new(socket, ssl_context)

data/lib/http/uri.rb

@@ -37,6 +37,9 @@ module HTTP
     # @private
     HTTPS_SCHEME = "https"
 
+    # @private
+    PERCENT_ENCODE = /[^\x21-\x7E]+/.freeze
+
     # @private
     NORMALIZER = lambda do |uri|
       uri = HTTP::URI.parse uri
@@ -44,8 +47,8 @@ module HTTP
       HTTP::URI.new(
         :scheme    => uri.normalized_scheme,
         :authority => uri.normalized_authority,
-        :path      => uri.normalized_path,
-        :query     => uri.query,
+        :path      => uri.path.empty? ? "/" : percent_encode(Addressable::URI.normalize_path(uri.path)),
+        :query     => percent_encode(uri.query),
         :fragment  => uri.normalized_fragment
       )
     end
@@ -71,6 +74,19 @@ module HTTP
       Addressable::URI.form_encode(form_values, sort)
     end
 
+    # Percent-encode all characters matching a regular expression.
+    #
+    # @param [String] string raw string
+    #
+    # @return [String] encoded value
+    #
+    # @private
+    def self.percent_encode(string)
+      string&.gsub(PERCENT_ENCODE) do |substr|
+        substr.encode(Encoding::UTF_8).bytes.map { |c| format("%%%02X", c) }.join
+      end
+    end
+
     # Creates an HTTP::URI instance from the given options
     #
     # @param [Hash, Addressable::URI] options_or_uri

data/lib/http/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTP
-  VERSION = "5.1.1"
+  VERSION = "5.3.0"
 end

data/lib/http.rb

@@ -6,6 +6,7 @@ require "http/timeout/per_operation"
 require "http/timeout/global"
 require "http/chainable"
 require "http/client"
+require "http/retriable/client"
 require "http/connection"
 require "http/options"
 require "http/feature"

data/spec/lib/http/client_spec.rb

@@ -261,6 +261,7 @@ RSpec.describe HTTP::Client do
 
       expect(HTTP::Request).to receive(:new) do |opts|
         expect(opts[:body]).to eq '{"foo":"bar"}'
+        expect(opts[:headers]["Content-Type"]).to eq "application/json; charset=utf-8"
       end
 
       client.get("http://example.com/", :json => {:foo => :bar})

data/spec/lib/http/connection_spec.rb

@@ -8,11 +8,31 @@ RSpec.describe HTTP::Connection do
       :headers => {}
     )
   end
-  let(:socket) { double(:connect => nil) }
+  let(:socket) { double(:connect => nil, :close => nil) }
   let(:timeout_class) { double(:new => socket) }
   let(:opts) { HTTP::Options.new(:timeout_class => timeout_class) }
   let(:connection) { HTTP::Connection.new(req, opts) }
 
+  describe "#initialize times out" do
+    let(:req) do
+      HTTP::Request.new(
+        :verb    => :get,
+        :uri     => "https://example.com/",
+        :headers => {}
+      )
+    end
+
+    before do
+      expect(socket).to receive(:start_tls).and_raise(HTTP::TimeoutError)
+      expect(socket).to receive(:closed?) { false }
+      expect(socket).to receive(:close)
+    end
+
+    it "closes the connection" do
+      expect { connection }.to raise_error(HTTP::TimeoutError)
+    end
+  end
+
   describe "#read_headers!" do
     before do
       connection.instance_variable_set(:@pending_response, true)
@@ -58,9 +78,11 @@ RSpec.describe HTTP::Connection do
       connection.read_headers!
       buffer = String.new
       while (s = connection.readpartial(3))
+        expect(connection.finished_request?).to be false if s != ""
         buffer << s
       end
       expect(buffer).to eq "1234567890"
+      expect(connection.finished_request?).to be true
     end
   end
 end

data/spec/lib/http/features/instrumentation_spec.rb

@@ -59,4 +59,23 @@ RSpec.describe HTTP::Features::Instrumentation do
       expect(instrumenter.output[:finish]).to eq(:response => response)
     end
   end
+
+  describe "logging errors" do
+    let(:request) do
+      HTTP::Request.new(
+        :verb    => :post,
+        :uri     => "https://example.com/",
+        :headers => {:accept => "application/json"},
+        :body    => '{"hello": "world!"}'
+      )
+    end
+
+    let(:error) { HTTP::TimeoutError.new }
+
+    it "should log the error" do
+      feature.on_error(request, error)
+
+      expect(instrumenter.output[:finish]).to eq(:request => request, :error => error)
+    end
+  end
 end

data/spec/lib/http/features/raise_error_spec.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+RSpec.describe HTTP::Features::RaiseError do
+  subject(:feature) { described_class.new(ignore: ignore) }
+
+  let(:connection) { double }
+  let(:status) { 200 }
+  let(:ignore) { [] }
+
+  describe "#wrap_response" do
+    subject(:result) { feature.wrap_response(response) }
+
+    let(:response) do
+      HTTP::Response.new(
+        version:    "1.1",
+        status:     status,
+        headers:    {},
+        connection: connection,
+        request:    HTTP::Request.new(verb: :get, uri: "https://example.com")
+      )
+    end
+
+    context "when status is 200" do
+      it "returns original request" do
+        expect(result).to be response
+      end
+    end
+
+    context "when status is 399" do
+      let(:status) { 399 }
+
+      it "returns original request" do
+        expect(result).to be response
+      end
+    end
+
+    context "when status is 400" do
+      let(:status) { 400 }
+
+      it "raises" do
+        expect { result }.to raise_error(HTTP::StatusError, "Unexpected status code 400")
+      end
+    end
+
+    context "when status is 599" do
+      let(:status) { 599 }
+
+      it "raises" do
+        expect { result }.to raise_error(HTTP::StatusError, "Unexpected status code 599")
+      end
+    end
+
+    context "when error status is ignored" do
+      let(:status) { 500 }
+      let(:ignore) { [500] }
+
+      it "returns original request" do
+        expect(result).to be response
+      end
+    end
+  end
+end