http 5.1.1 → 5.3.0

data/spec/lib/http/headers/normalizer_spec.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+RSpec.describe HTTP::Headers::Normalizer do
+  subject(:normalizer) { described_class.new }
+
+  include_context RSpec::Memory
+
+  describe "#call" do
+    it "normalizes the header" do
+      expect(normalizer.call("content_type")).to eq "Content-Type"
+    end
+
+    it "returns a non-frozen string" do
+      expect(normalizer.call("content_type")).not_to be_frozen
+    end
+
+    it "evicts the oldest item when cache is full" do
+      max_headers = (1..described_class::Cache::MAX_SIZE).map { |i| "Header#{i}" }
+      max_headers.each { |header| normalizer.call(header) }
+      normalizer.call("New-Header")
+      cache_store = normalizer.instance_variable_get(:@cache).instance_variable_get(:@store)
+      expect(cache_store.keys).to eq(max_headers[1..] + ["New-Header"])
+    end
+
+    it "retuns mutable strings" do
+      normalized_headers = Array.new(3) { normalizer.call("content_type") }
+
+      expect(normalized_headers)
+        .to satisfy { |arr| arr.uniq.size == 1 }
+        .and(satisfy { |arr| arr.map(&:object_id).uniq.size == normalized_headers.size })
+        .and(satisfy { |arr| arr.none?(&:frozen?) })
+    end
+
+    it "allocates minimal memory for normalization of the same header" do
+      normalizer.call("accept") # XXX: Ensure normalizer is pre-allocated
+
+      # On first call it is expected to allocate during normalization
+      expect { normalizer.call("content_type") }.to limit_allocations(
+        Array     => 1,
+        MatchData => 1,
+        String    => 6
+      )
+
+      # On subsequent call it is expected to only allocate copy of a cached string
+      expect { normalizer.call("content_type") }.to limit_allocations(
+        Array     => 0,
+        MatchData => 0,
+        String    => 1
+      )
+    end
+  end
+end

data/spec/lib/http/options/headers_spec.rb

@@ -14,7 +14,11 @@ RSpec.describe HTTP::Options, "headers" do
   end
 
   it "accepts any object that respond to :to_hash" do
-    x = Struct.new(:to_hash).new("accept" => "json")
+    x = if RUBY_VERSION >= "3.2.0"
+          Data.define(:to_hash).new(:to_hash => { "accept" => "json" })
+        else
+          Struct.new(:to_hash).new({ "accept" => "json" })
+        end
     expect(opts.with_headers(x).headers["accept"]).to eq("json")
   end
 end

data/spec/lib/http/redirector_spec.rb

@@ -115,12 +115,13 @@ RSpec.describe HTTP::Redirector do
         expect(req_cookie).to eq request_cookies.shift
         hops.shift
       end
+
       expect(res.to_s).to eq "bar"
-      cookies = res.cookies.cookies.to_h { |c| [c.name, c.value] }
-      expect(cookies["foo"]).to eq "42"
-      expect(cookies["bar"]).to eq "53"
-      expect(cookies["baz"]).to eq "65"
-      expect(cookies["deleted"]).to eq nil
+      expect(res.cookies.cookies.to_h { |c| [c.name, c.value] }).to eq({
+        "foo" => "42",
+        "bar" => "53",
+        "baz" => "65"
+      })
     end
 
     it "returns original cookies in response" do

data/spec/lib/http/retriable/delay_calculator_spec.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+RSpec.describe HTTP::Retriable::DelayCalculator do
+  let(:response) do
+    HTTP::Response.new(
+      status:  200,
+      version: "1.1",
+      headers: {},
+      body:    "Hello world!",
+      request: HTTP::Request.new(verb: :get, uri: "http://example.com")
+    )
+  end
+
+  def call_delay(iterations, **options)
+    described_class.new(options).call(iterations, response)
+  end
+
+  def call_retry_header(value, **options)
+    response.headers["Retry-After"] = value
+    described_class.new(options).call(rand(1...100), response)
+  end
+
+  it "prevents negative sleep time" do
+    expect(call_delay(20, delay: -20)).to eq 0
+  end
+
+  it "backs off exponentially" do
+    expect(call_delay(1)).to be_between 0, 1
+    expect(call_delay(2)).to be_between 1, 2
+    expect(call_delay(3)).to be_between 3, 4
+    expect(call_delay(4)).to be_between 7, 8
+    expect(call_delay(5)).to be_between 15, 16
+  end
+
+  it "can have a maximum wait time" do
+    expect(call_delay(1, max_delay: 5)).to be_between 0, 1
+    expect(call_delay(5, max_delay: 5)).to eq 5
+  end
+
+  it "respects Retry-After headers as integer" do
+    delay_time = rand(6...2500)
+    header_value = delay_time.to_s
+    expect(call_retry_header(header_value)).to eq delay_time
+    expect(call_retry_header(header_value, max_delay: 5)).to eq 5
+  end
+
+  it "respects Retry-After headers as rfc2822 timestamp" do
+    delay_time = rand(6...2500)
+    header_value = (Time.now.gmtime + delay_time).to_datetime.rfc2822.sub("+0000", "GMT")
+    expect(call_retry_header(header_value)).to be_within(1).of(delay_time)
+    expect(call_retry_header(header_value, max_delay: 5)).to eq 5
+  end
+
+  it "respects Retry-After headers as rfc2822 timestamp in the past" do
+    delay_time = rand(6...2500)
+    header_value = (Time.now.gmtime - delay_time).to_datetime.rfc2822.sub("+0000", "GMT")
+    expect(call_retry_header(header_value)).to eq 0
+  end
+
+  it "does not error on invalid Retry-After header" do
+    [ # invalid strings
+      "This is a string with a number 5 in it",
+      "8 Eight is the first digit in this string",
+      "This is a string with a #{Time.now.gmtime.to_datetime.rfc2822} timestamp in it"
+    ].each do |header_value|
+      expect(call_retry_header(header_value)).to eq 0
+    end
+  end
+end

data/spec/lib/http/retriable/performer_spec.rb

@@ -0,0 +1,302 @@
+# frozen_string_literal: true
+
+RSpec.describe HTTP::Retriable::Performer do
+  let(:client) do
+    HTTP::Client.new
+  end
+
+  let(:response) do
+    HTTP::Response.new(
+      status:  200,
+      version: "1.1",
+      headers: {},
+      body:    "Hello world!",
+      request: request
+    )
+  end
+
+  let(:request) do
+    HTTP::Request.new(
+      verb: :get,
+      uri:  "http://example.com"
+    )
+  end
+
+  let(:perform_spy) { { counter: 0 } }
+  let(:counter_spy) { perform_spy[:counter] }
+
+  before do
+    stub_const("CustomException", Class.new(StandardError))
+  end
+
+  def perform(options = {}, client_arg = client, request_arg = request, &block)
+    # by explicitly overwriting the default delay, we make a much faster test suite
+    default_options = { delay: 0 }
+    options = default_options.merge(options)
+
+    HTTP::Retriable::Performer
+      .new(options)
+      .perform(client_arg, request_arg) do
+        perform_spy[:counter] += 1
+        block ? yield : response
+      end
+  end
+
+  def measure_wait
+    t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    result = yield
+    t2 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    [t2 - t1, result]
+  end
+
+  describe "#perform" do
+    describe "expected exception" do
+      it "retries the request" do
+        expect do
+          perform(exceptions: [CustomException], tries: 2) do
+            raise CustomException
+          end
+        end.to raise_error HTTP::OutOfRetriesError
+
+        expect(counter_spy).to eq 2
+      end
+    end
+
+    describe "unexpected exception" do
+      it "does not retry the request" do
+        expect do
+          perform(exceptions: [], tries: 2) do
+            raise CustomException
+          end
+        end.to raise_error CustomException
+
+        expect(counter_spy).to eq 1
+      end
+    end
+
+    describe "expected status codes" do
+      def response(**options)
+        HTTP::Response.new(
+          {
+            status:  200,
+            version: "1.1",
+            headers: {},
+            body:    "Hello world!",
+            request: request
+          }.merge(options)
+        )
+      end
+
+      it "retries the request" do
+        expect do
+          perform(retry_statuses: [200], tries: 2)
+        end.to raise_error HTTP::OutOfRetriesError
+
+        expect(counter_spy).to eq 2
+      end
+
+      describe "status codes can be expressed in many ways" do
+        [
+          301,
+          [200, 301, 485],
+          250...400,
+          [250...Float::INFINITY],
+          ->(status_code) { status_code == 301 },
+          [->(status_code) { status_code == 301 }]
+        ].each do |retry_statuses|
+          it retry_statuses.to_s do
+            expect do
+              perform(retry_statuses: retry_statuses, tries: 2) do
+                response(status: 301)
+              end
+            end.to raise_error HTTP::OutOfRetriesError
+          end
+        end
+      end
+    end
+
+    describe "unexpected status code" do
+      it "does not retry the request" do
+        expect(
+          perform(retry_statuses: [], tries: 2)
+        ).to eq response
+
+        expect(counter_spy).to eq 1
+      end
+    end
+
+    describe "on_retry callback" do
+      it "calls the on_retry callback on each retry with exception" do
+        callback_call_spy = 0
+
+        callback_spy = proc do |callback_request, error, callback_response|
+          expect(callback_request).to eq request
+          expect(error).to be_a HTTP::TimeoutError
+          expect(callback_response).to be_nil
+          callback_call_spy += 1
+        end
+
+        expect do
+          perform(tries: 3, on_retry: callback_spy) do
+            raise HTTP::TimeoutError
+          end
+        end.to raise_error HTTP::OutOfRetriesError
+
+        expect(callback_call_spy).to eq 2
+      end
+
+      it "calls the on_retry callback on each retry with response" do
+        callback_call_spy = 0
+
+        callback_spy = proc do |callback_request, error, callback_response|
+          expect(callback_request).to eq request
+          expect(error).to be_nil
+          expect(callback_response).to be response
+          callback_call_spy += 1
+        end
+
+        expect do
+          perform(retry_statuses: [200], tries: 3, on_retry: callback_spy)
+        end.to raise_error HTTP::OutOfRetriesError
+
+        expect(callback_call_spy).to eq 2
+      end
+    end
+
+    describe "delay option" do
+      let(:timing_slack) { 0.05 }
+
+      it "can be a positive number" do
+        time, = measure_wait do
+          perform(delay: 0.1, tries: 3, should_retry: ->(*) { true })
+        rescue HTTP::OutOfRetriesError
+        end
+        expect(time).to be_within(timing_slack).of(0.2)
+      end
+
+      it "can be a proc number" do
+        time, = measure_wait do
+          perform(delay: ->(attempt) { attempt / 10.0 }, tries: 3, should_retry: ->(*) { true })
+        rescue HTTP::OutOfRetriesError
+        end
+        expect(time).to be_within(timing_slack).of(0.3)
+      end
+
+      it "receives correct retry number when a proc" do
+        retry_count = 0
+        retry_proc = proc do |attempt|
+          expect(attempt).to eq(retry_count).and(be > 0)
+          0
+        end
+        begin
+          perform(delay: retry_proc, should_retry: ->(*) { true }) do
+            retry_count += 1
+            response
+          end
+        rescue HTTP::OutOfRetriesError
+        end
+      end
+    end
+
+    describe "should_retry option" do
+      it "decides if the request should be retried" do # rubocop:disable RSpec/MultipleExpectations
+        retry_proc = proc do |req, err, res, attempt|
+          expect(req).to eq request
+          if res
+            expect(err).to be_nil
+            expect(res).to be response
+          else
+            expect(err).to be_a CustomException
+            expect(res).to be_nil
+          end
+
+          attempt < 5
+        end
+
+        begin
+          perform(should_retry: retry_proc) do
+            rand < 0.5 ? response : raise(CustomException)
+          end
+        rescue CustomException
+        end
+
+        expect(counter_spy).to eq 5
+      end
+
+      it "raises the original error if not retryable" do
+        retry_proc = ->(*) { false }
+
+        expect do
+          perform(should_retry: retry_proc) do
+            raise CustomException
+          end
+        end.to raise_error CustomException
+
+        expect(counter_spy).to eq 1
+      end
+
+      it "raises HTTP::OutOfRetriesError if retryable" do
+        retry_proc = ->(*) { true }
+
+        expect do
+          perform(should_retry: retry_proc) do
+            raise CustomException
+          end
+        end.to raise_error HTTP::OutOfRetriesError
+
+        expect(counter_spy).to eq 5
+      end
+    end
+  end
+
+  describe "connection closing" do
+    let(:client) { double(:client) }
+
+    it "does not close the connection if we get a propper response" do
+      expect(client).not_to receive(:close)
+      perform
+    end
+
+    it "closes the connection after each raiseed attempt" do
+      expect(client).to receive(:close).exactly(3).times
+      begin
+        perform(should_retry: ->(*) { true }, tries: 3)
+      rescue HTTP::OutOfRetriesError
+      end
+    end
+
+    it "closes the connection on an unexpected exception" do
+      expect(client).to receive(:close)
+      begin
+        perform do
+          raise CustomException
+        end
+      rescue CustomException
+      end
+    end
+  end
+
+  describe HTTP::OutOfRetriesError do
+    it "has the original exception as a cause if available" do
+      err = nil
+      begin
+        perform(exceptions: [CustomException]) do
+          raise CustomException
+        end
+      rescue described_class => e
+        err = e
+      end
+      expect(err.cause).to be_a CustomException
+    end
+
+    it "has the last raiseed response as an attribute" do
+      err = nil
+      begin
+        perform(should_retry: ->(*) { true })
+      rescue described_class => e
+        err = e
+      end
+      expect(err.response).to be response
+    end
+  end
+end

data/spec/lib/http/uri/normalizer_spec.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+RSpec.describe HTTP::URI::NORMALIZER do
+  describe "scheme" do
+    it "lower-cases scheme" do
+      expect(HTTP::URI::NORMALIZER.call("HttP://example.com").scheme).to eq "http"
+    end
+  end
+
+  describe "hostname" do
+    it "lower-cases hostname" do
+      expect(HTTP::URI::NORMALIZER.call("http://EXAMPLE.com").host).to eq "example.com"
+    end
+
+    it "decodes percent-encoded hostname" do
+      expect(HTTP::URI::NORMALIZER.call("http://ex%61mple.com").host).to eq "example.com"
+    end
+
+    it "removes trailing period in hostname" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com.").host).to eq "example.com"
+    end
+
+    it "IDN-encodes non-ASCII hostname" do
+      expect(HTTP::URI::NORMALIZER.call("http://exämple.com").host).to eq "xn--exmple-cua.com"
+    end
+  end
+
+  describe "path" do
+    it "ensures path is not empty" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com").path).to eq "/"
+    end
+
+    it "preserves double slashes in path" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com//a///b").path).to eq "//a///b"
+    end
+
+    it "resolves single-dot segments in path" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/a/./b").path).to eq "/a/b"
+    end
+
+    it "resolves double-dot segments in path" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/a/b/../c").path).to eq "/a/c"
+    end
+
+    it "resolves leading double-dot segments in path" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/../a/b").path).to eq "/a/b"
+    end
+
+    it "percent-encodes control characters in path" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/\x00\x7F\n").path).to eq "/%00%7F%0A"
+    end
+
+    it "percent-encodes space in path" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/a b").path).to eq "/a%20b"
+    end
+
+    it "percent-encodes non-ASCII characters in path" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/キョ").path).to eq "/%E3%82%AD%E3%83%A7"
+    end
+
+    it "does not percent-encode non-special characters in path" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/~.-_!$&()*,;=:@{}").path).to eq "/~.-_!$&()*,;=:@{}"
+    end
+
+    it "preserves escape sequences in path" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/%41").path).to eq "/%41"
+    end
+  end
+
+  describe "query" do
+    it "allows no query" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com").query).to be_nil
+    end
+
+    it "percent-encodes control characters in query" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/?\x00\x7F\n").query).to eq "%00%7F%0A"
+    end
+
+    it "percent-encodes space in query" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/?a b").query).to eq "a%20b"
+    end
+
+    it "percent-encodes non-ASCII characters in query" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com?キョ").query).to eq "%E3%82%AD%E3%83%A7"
+    end
+
+    it "does not percent-encode non-special characters in query" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/?~.-_!$&()*,;=:@{}?").query).to eq "~.-_!$&()*,;=:@{}?"
+    end
+
+    it "preserves escape sequences in query" do
+      expect(HTTP::URI::NORMALIZER.call("http://example.com/?%41").query).to eq "%41"
+    end
+  end
+end

data/spec/lib/http_spec.rb

@@ -152,6 +152,35 @@ RSpec.describe HTTP do
     end
   end
 
+  describe ".retry" do
+    it "ensure endpoint counts retries" do
+      expect(HTTP.get("#{dummy.endpoint}/retry-2").to_s).to eq "retried 1x"
+      expect(HTTP.get("#{dummy.endpoint}/retry-2").to_s).to eq "retried 2x"
+    end
+
+    it "retries the request" do
+      response = HTTP.retriable(delay: 0, retry_statuses: 500...600).get "#{dummy.endpoint}/retry-2"
+      expect(response.to_s).to eq "retried 2x"
+    end
+
+    it "retries the request and gives us access to the failed requests" do
+      err = nil
+      retry_callback = ->(_, _, res) { expect(res.to_s).to match(/^retried \dx$/) }
+      begin
+        HTTP.retriable(
+          should_retry: ->(*) { true },
+          tries:        3,
+          delay:        0,
+          on_retry:     retry_callback
+        ).get "#{dummy.endpoint}/retry-2"
+      rescue HTTP::Error => e
+        err = e
+      end
+
+      expect(err.response.to_s).to eq "retried 3x"
+    end
+  end
+
   context "posting forms to resources" do
     it "is easy" do
       response = HTTP.post "#{dummy.endpoint}/form", :form => {:example => "testing-form"}
@@ -460,20 +489,37 @@ RSpec.describe HTTP do
 
     context "with :normalize_uri" do
       it "normalizes URI" do
-        response = HTTP.get "#{dummy.endpoint}/hello world"
+        response = HTTP.get "#{dummy.endpoint}/héllö-wörld"
         expect(response.to_s).to eq("hello world")
       end
 
       it "uses the custom URI Normalizer method" do
         client = HTTP.use(:normalize_uri => {:normalizer => :itself.to_proc})
-        response = client.get("#{dummy.endpoint}/hello world")
+        response = client.get("#{dummy.endpoint}/héllö-wörld")
         expect(response.status).to eq(400)
       end
 
+      it "raises if custom URI Normalizer returns invalid path" do
+        client = HTTP.use(:normalize_uri => {:normalizer => :itself.to_proc})
+        expect { client.get("#{dummy.endpoint}/hello\nworld") }.
+          to raise_error HTTP::RequestError, 'Invalid request URI: "/hello\nworld"'
+      end
+
+      it "raises if custom URI Normalizer returns invalid host" do
+        normalizer = lambda do |uri|
+          uri.port = nil
+          uri.instance_variable_set(:@host, "example\ncom")
+          uri
+        end
+        client = HTTP.use(:normalize_uri => {:normalizer => normalizer})
+        expect { client.get(dummy.endpoint) }.
+          to raise_error HTTP::RequestError, 'Invalid host: "example\ncom"'
+      end
+
       it "uses the default URI normalizer" do
         client = HTTP.use :normalize_uri
         expect(HTTP::URI::NORMALIZER).to receive(:call).and_call_original
-        response = client.get("#{dummy.endpoint}/hello world")
+        response = client.get("#{dummy.endpoint}/héllö-wörld")
         expect(response.to_s).to eq("hello world")
       end
     end

data/spec/spec_helper.rb

@@ -5,6 +5,7 @@ require_relative "./support/fuubar" unless ENV["CI"]
 
 require "http"
 require "rspec/its"
+require "rspec/memory"
 require "support/capture_warning"
 require "support/fakeio"