http 0.7.4 → 0.8.0.pre

data/lib/http/response/string_body.rb

@@ -0,0 +1,53 @@
+module HTTP
+  class Response
+    # A Body class that wraps a String, rather than a the client
+    # object.
+    class StringBody
+      include Enumerable
+      extend Forwardable
+
+      # @return [String,nil] the next `size` octets part of the
+      # body, or nil if whole body has already been read.
+      def readpartial(size = @contents.length)
+        stream!
+        return nil if @streaming_offset >= @contents.length
+
+        @contents[@streaming_offset, size].tap do |part|
+          @streaming_offset += (part.length + 1)
+        end
+      end
+
+      # Iterate over the body, allowing it to be enumerable
+      def each
+        yield @contents
+      end
+
+      # @return [String] eagerly consume the entire body as a string
+      def to_s
+        @contents
+      end
+      alias_method :to_str, :to_s
+
+      def_delegator :@contents, :empty?
+
+      # Assert that the body is actively being streamed
+      def stream!
+        fail StateError, "body has already been consumed" if @streaming == false
+        @streaming = true
+      end
+
+      # Easier to interpret string inspect
+      def inspect
+        "#<#{self.class}:#{object_id.to_s(16)}>"
+      end
+
+      protected
+
+      def initialize(contents)
+        @contents = contents
+        @streaming = nil
+        @streaming_offset = 0
+      end
+    end
+  end
+end

data/lib/http/version.rb

@@ -1,3 +1,3 @@
 module HTTP
-  VERSION = '0.7.4'
+  VERSION = "0.8.0.pre"
 end

data/spec/lib/http/cache/headers_spec.rb

@@ -0,0 +1,77 @@
+RSpec.describe HTTP::Cache::Headers do
+  subject(:cache_headers) { described_class.new headers }
+
+  describe ".new" do
+    it "accepts instance of HTTP::Headers" do
+      expect { described_class.new HTTP::Headers.new }.not_to raise_error
+    end
+
+    it "it rejects any object that does not respond to #headers" do
+      expect { described_class.new double }.to raise_error HTTP::Error
+    end
+  end
+
+  context "with <Cache-Control: private>" do
+    let(:headers) { {"Cache-Control" => "private"} }
+    it { is_expected.to be_private }
+  end
+
+  context "with <Cache-Control: public>" do
+    let(:headers) { {"Cache-Control" => "public"} }
+    it { is_expected.to be_public }
+  end
+
+  context "with <Cache-Control: no-cache>" do
+    let(:headers) { {"Cache-Control" => "no-cache"} }
+    it { is_expected.to be_no_cache }
+  end
+
+  context "with <Cache-Control: no-store>" do
+    let(:headers) { {"Cache-Control" => "no-store"} }
+    it { is_expected.to be_no_store }
+  end
+
+  describe "#max_age" do
+    subject { cache_headers.max_age }
+
+    context "with <Cache-Control: max-age=100>" do
+      let(:headers) { {"Cache-Control" => "max-age=100"} }
+      it { is_expected.to eq 100 }
+    end
+
+    context "with <Expires: {100 seconds from now}>" do
+      let(:headers) { {"Expires" => (Time.now + 100).httpdate} }
+      it { is_expected.to be_within(1).of(100) }
+    end
+
+    context "with <Expires: {100 seconds before now}>" do
+      let(:headers) { {"Expires" => (Time.now - 100).httpdate} }
+      it { is_expected.to eq 0 }
+    end
+
+    context "with <Expires: -1>" do
+      let(:headers) { {"Expires" => "-1"} }
+      it { is_expected.to eq 0 }
+    end
+  end
+
+  context "with <Vary: *>" do
+    let(:headers) { {"Vary" => "*"} }
+    it { is_expected.to be_vary_star }
+  end
+
+  context "with no cache related headers" do
+    let(:headers) { {} }
+
+    it { is_expected.not_to be_private }
+    it { is_expected.not_to be_public }
+    it { is_expected.not_to be_no_cache }
+    it { is_expected.not_to be_no_store }
+    it { is_expected.not_to be_vary_star }
+
+    describe "#max_age" do
+      subject { cache_headers.max_age }
+      it { is_expected.to eq Float::INFINITY }
+    end
+  end
+end

data/spec/lib/http/cache_spec.rb

@@ -0,0 +1,182 @@
+require "support/dummy_server"
+require "http/cache"
+
+RSpec.describe HTTP::Cache do
+  describe "creation" do
+    subject { described_class }
+
+    it "allows metastore and entitystore" do
+      expect(subject.new(:metastore => "heap:/", :entitystore => "heap:/"))
+        .to be_kind_of HTTP::Cache
+    end
+  end
+
+  let(:opts) { options }
+  let(:sn) { SecureRandom.urlsafe_base64(3) }
+  let(:request) { HTTP::Request.new(:get, "http://example.com/#{sn}") }
+
+  let(:origin_response) do
+    HTTP::Response.new(200,
+                       "http/1.1",
+                       {"Cache-Control" => "private"},
+                       "origin")
+  end
+
+  subject { described_class.new(:metastore => "heap:/", :entitystore => "heap:/") }
+
+  describe "#perform" do
+    it "calls request_performer blocck when cache miss" do
+      expect do |b|
+        subject.perform(request, opts) do |*args|
+          b.to_proc.call(*args)
+          origin_response
+        end
+      end.to yield_with_args(request, opts)
+    end
+
+    context "cache hit" do
+      it "does not call request_performer block" do
+        subject.perform(request, opts) do |*_t|
+          origin_response
+        end
+
+        expect { |b| subject.perform(request, opts, &b) }.not_to yield_control
+      end
+    end
+  end
+
+  context "empty cache, cacheable request, cacheable response" do
+    let!(:response) { subject.perform(request, opts) { origin_response } }
+
+    it "returns origin servers response" do
+      expect(response).to eq origin_response
+    end
+  end
+
+  context "cache by-passing request, cacheable response" do
+    let(:request) do
+      headers = {"Cache-Control" => "no-cache"}
+      HTTP::Request.new(:get, "http://example.com/", headers)
+    end
+    let!(:response) { subject.perform(request, opts) { origin_response } }
+
+    it "returns origin servers response" do
+      expect(response).to eq origin_response
+    end
+  end
+
+  context "empty cache, cacheable request, 'no-cache' response" do
+    let(:origin_response) do
+      HTTP::Response.new(200,
+                         "http/1.1",
+                         {"Cache-Control" => "no-store"},
+                         "")
+    end
+    let!(:response) { subject.perform(request, opts) { origin_response } }
+
+    it "returns origin servers response" do
+      expect(response).to eq origin_response
+    end
+  end
+
+  context "empty cache, cacheable request, 'no-store' response" do
+    let(:origin_response) do
+      HTTP::Response.new(200,
+                         "http/1.1",
+                         {"Cache-Control" => "no-store"},
+                         "")
+    end
+    let!(:response) { subject.perform(request, opts) { origin_response } }
+
+    it "returns origin servers response" do
+      expect(response).to eq origin_response
+    end
+  end
+
+  context "warm cache, cacheable request, cacheable response" do
+    let(:cached_response) do
+      build_cached_response(200,
+                            "1.1",
+                            {"Cache-Control" => "max-age=100"},
+                            "cached")
+    end
+    before do
+      subject.perform(request, opts) { cached_response }
+    end
+
+    let(:response) { subject.perform(request, opts) { origin_response } }
+
+    it "returns cached response" do
+      expect(response.body.to_s).to eq cached_response.body.to_s
+    end
+  end
+
+  context "stale cache, cacheable request, cacheable response" do
+    let(:cached_response) do
+      build_cached_response(200,
+                            "1.1",
+                            {"Cache-Control" => "private, max-age=1",
+                             "Date" => (Time.now - 2).httpdate},
+                            "cached") do |t|
+        t.requested_at = (Time.now - 2)
+      end
+    end
+    before do
+      subject.perform(request, opts) { cached_response }
+    end
+    let(:response) { subject.perform(request, opts) { origin_response } }
+
+    it "returns origin servers response" do
+      expect(response.body.to_s).to eq origin_response.body.to_s
+    end
+  end
+
+  context "stale cache, cacheable request, not modified response" do
+    let(:cached_response) do
+      build_cached_response(200,
+                            "http/1.1",
+                            {"Cache-Control" => "private, max-age=1",
+                             "Etag" => "foo",
+                             "Date" => (Time.now - 2).httpdate},
+                            "") do |x|
+        x.requested_at = (Time.now - 2)
+      end
+    end
+    before do
+      subject.perform(request, opts) { cached_response }
+    end
+
+    let(:origin_response) { HTTP::Response.new(304, "http/1.1", {}, "") }
+    let(:response) { subject.perform(request, opts) { origin_response } }
+
+    it "makes request with conditional request headers" do
+      subject.perform(request, opts) do |actual_request, _|
+        expect(actual_request.headers["If-None-Match"])
+          .to eq cached_response.headers["Etag"]
+        expect(actual_request.headers["If-Modified-Since"])
+          .to eq cached_response.headers["Last-Modified"]
+
+        origin_response
+      end
+    end
+
+    it "returns cached servers response" do
+      expect(response.body.to_s).to eq cached_response.body.to_s
+    end
+  end
+
+  let(:cached_response) { nil } # cold cache by default
+
+  def build_cached_response(*args)
+    r = HTTP::Response.new(*args).caching
+    r.requested_at = r.received_at = Time.now
+
+    yield r if block_given?
+
+    r
+  end
+
+  def options
+    HTTP::Options.new
+  end
+end

data/spec/lib/http/client_spec.rb

@@ -1,12 +1,13 @@
-require 'support/dummy_server'
+require "support/connection_reuse_shared"
+require "support/dummy_server"
+require "http/cache"
 
 RSpec.describe HTTP::Client do
-  let(:test_endpoint)  { "http://#{ExampleServer::ADDR}" }
   run_server(:dummy) { DummyServer.new }
   run_server(:dummy_ssl) { DummyServer.new(:ssl => true) }
 
   StubbedClient = Class.new(HTTP::Client) do
-    def perform(request, options)
+    def make_request(request, options)
       stubs.fetch(request.uri.to_s) { super(request, options) }
     end
 
@@ -21,105 +22,125 @@ RSpec.describe HTTP::Client do
   end
 
   def redirect_response(location, status = 302)
-    HTTP::Response.new(status, '1.1', {'Location' => location}, '')
+    HTTP::Response.new(status, "1.1", {"Location" => location}, "")
   end
 
   def simple_response(body, status = 200)
-    HTTP::Response.new(status, '1.1', {}, body)
+    HTTP::Response.new(status, "1.1", {}, body)
   end
 
-  describe 'following redirects' do
-    it 'returns response of new location' do
+  describe "following redirects" do
+    it "returns response of new location" do
       client = StubbedClient.new(:follow => true).stub(
-        'http://example.com/'     => redirect_response('http://example.com/blog'),
-        'http://example.com/blog' => simple_response('OK')
+        "http://example.com/"     => redirect_response("http://example.com/blog"),
+        "http://example.com/blog" => simple_response("OK")
       )
 
-      expect(client.get('http://example.com/').to_s).to eq 'OK'
+      expect(client.get("http://example.com/").to_s).to eq "OK"
     end
 
-    it 'prepends previous request uri scheme and host if needed' do
+    it "prepends previous request uri scheme and host if needed" do
       client = StubbedClient.new(:follow => true).stub(
-        'http://example.com/'           => redirect_response('/index'),
-        'http://example.com/index'      => redirect_response('/index.html'),
-        'http://example.com/index.html' => simple_response('OK')
+        "http://example.com/"           => redirect_response("/index"),
+        "http://example.com/index"      => redirect_response("/index.html"),
+        "http://example.com/index.html" => simple_response("OK")
       )
 
-      expect(client.get('http://example.com/').to_s).to eq 'OK'
+      expect(client.get("http://example.com/").to_s).to eq "OK"
     end
 
-    it 'fails upon endless redirects' do
+    it "fails upon endless redirects" do
       client = StubbedClient.new(:follow => true).stub(
-        'http://example.com/' => redirect_response('/')
+        "http://example.com/" => redirect_response("/")
       )
 
-      expect { client.get('http://example.com/') }
+      expect { client.get("http://example.com/") }
         .to raise_error(HTTP::Redirector::EndlessRedirectError)
     end
 
-    it 'fails if max amount of hops reached' do
+    it "fails if max amount of hops reached" do
       client = StubbedClient.new(:follow => 5).stub(
-        'http://example.com/'  => redirect_response('/1'),
-        'http://example.com/1' => redirect_response('/2'),
-        'http://example.com/2' => redirect_response('/3'),
-        'http://example.com/3' => redirect_response('/4'),
-        'http://example.com/4' => redirect_response('/5'),
-        'http://example.com/5' => redirect_response('/6'),
-        'http://example.com/6' => simple_response('OK')
+        "http://example.com/"  => redirect_response("/1"),
+        "http://example.com/1" => redirect_response("/2"),
+        "http://example.com/2" => redirect_response("/3"),
+        "http://example.com/3" => redirect_response("/4"),
+        "http://example.com/4" => redirect_response("/5"),
+        "http://example.com/5" => redirect_response("/6"),
+        "http://example.com/6" => simple_response("OK")
       )
 
-      expect { client.get('http://example.com/') }
+      expect { client.get("http://example.com/") }
         .to raise_error(HTTP::Redirector::TooManyRedirectsError)
     end
   end
 
-  describe 'parsing params' do
+  describe "caching" do
+    let(:sn) { SecureRandom.urlsafe_base64(3) }
+
+    it "returns cached responses if they exist" do
+      cached_response = simple_response("cached").caching
+      StubbedClient.new(:cache =>
+                        HTTP::Cache.new(:metastore => "heap:/", :entitystore => "heap:/"))
+        .stub("http://example.com/#{sn}" => cached_response)
+        .get("http://example.com/#{sn}")
+
+      # cache is now warm
+
+      client = StubbedClient.new(:cache => HTTP::Cache.new(:metastore => "heap:/", :entitystore => "heap:/"))
+               .stub("http://example.com/#{sn}" => simple_response("OK"))
+
+      expect(client.get("http://example.com/#{sn}").body.to_s)
+        .to eq cached_response.body.to_s
+    end
+  end
+
+  describe "parsing params" do
     let(:client) { HTTP::Client.new }
     before { allow(client).to receive :perform }
 
-    it 'accepts params within the provided URL' do
+    it "accepts params within the provided URL" do
       expect(HTTP::Request).to receive(:new) do |_, uri|
-        expect(CGI.parse uri.query).to eq('foo' => %w(bar))
+        expect(CGI.parse uri.query).to eq("foo" => %w(bar))
       end
 
-      client.get('http://example.com/?foo=bar')
+      client.get("http://example.com/?foo=bar")
     end
 
-    it 'combines GET params from the URI with the passed in params' do
+    it "combines GET params from the URI with the passed in params" do
       expect(HTTP::Request).to receive(:new) do |_, uri|
-        expect(CGI.parse uri.query).to eq('foo' => %w(bar), 'baz' => %w(quux))
+        expect(CGI.parse uri.query).to eq("foo" => %w(bar), "baz" => %w(quux))
       end
 
-      client.get('http://example.com/?foo=bar', :params => {:baz => 'quux'})
+      client.get("http://example.com/?foo=bar", :params => {:baz => "quux"})
     end
 
-    it 'merges duplicate values' do
+    it "merges duplicate values" do
       expect(HTTP::Request).to receive(:new) do |_, uri|
         expect(uri.query).to match(/^(a=1&a=2|a=2&a=1)$/)
       end
 
-      client.get('http://example.com/?a=1', :params => {:a => 2})
+      client.get("http://example.com/?a=1", :params => {:a => 2})
     end
 
-    it 'does not modifies query part if no params were given' do
+    it "does not modifies query part if no params were given" do
       expect(HTTP::Request).to receive(:new) do |_, uri|
-        expect(uri.query).to eq 'deadbeef'
+        expect(uri.query).to eq "deadbeef"
       end
 
-      client.get('http://example.com/?deadbeef')
+      client.get("http://example.com/?deadbeef")
     end
 
-    it 'does not corrupts index-less arrays' do
+    it "does not corrupts index-less arrays" do
       expect(HTTP::Request).to receive(:new) do |_, uri|
-        expect(CGI.parse uri.query).to eq 'a[]' => %w(b c), 'd' => %w(e)
+        expect(CGI.parse uri.query).to eq "a[]" => %w(b c), "d" => %w(e)
       end
 
-      client.get('http://example.com/?a[]=b&a[]=c', :params => {:d => 'e'})
+      client.get("http://example.com/?a[]=b&a[]=c", :params => {:d => "e"})
     end
   end
 
-  describe 'passing json' do
-    it 'encodes given object' do
+  describe "passing json" do
+    it "encodes given object" do
       client = HTTP::Client.new
       allow(client).to receive(:perform)
 
@@ -127,84 +148,91 @@ RSpec.describe HTTP::Client do
         expect(args.last).to eq('{"foo":"bar"}')
       end
 
-      client.get('http://example.com/', :json => {:foo => :bar})
+      client.get("http://example.com/", :json => {:foo => :bar})
     end
   end
 
-  describe '#request' do
-    context 'with explicitly given `Host` header' do
-      let(:headers) { {'Host' => 'another.example.com'} }
+  describe "#request" do
+    context "with explicitly given `Host` header" do
+      let(:headers) { {"Host" => "another.example.com"} }
       let(:client)  { described_class.new :headers => headers }
 
-      it 'keeps `Host` header as is' do
+      it "keeps `Host` header as is" do
         expect(client).to receive(:perform) do |req, _|
-          expect(req['Host']).to eq 'another.example.com'
+          expect(req["Host"]).to eq "another.example.com"
         end
 
-        client.request(:get, 'http://example.com/')
+        client.request(:get, "http://example.com/")
       end
     end
   end
 
-  describe 'SSL' do
+  include_context "handles shared connections" do
+    let(:reuse_conn) { nil }
+    let(:server) { dummy }
+    let(:client) { described_class.new(:persistent => reuse_conn) }
+  end
+
+  describe "SSL" do
+    let(:reuse_conn) { nil }
+
     let(:client) do
       described_class.new(
+        :persistent => reuse_conn,
         :ssl_context => OpenSSL::SSL::SSLContext.new.tap do |context|
           context.options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
 
           context.verify_mode = OpenSSL::SSL::VERIFY_PEER
-          context.ca_file = File.join(certs_dir, 'ca.crt')
+          context.ca_file = File.join(certs_dir, "ca.crt")
           context.cert = OpenSSL::X509::Certificate.new(
-            File.read(File.join(certs_dir, 'client.crt'))
+            File.read(File.join(certs_dir, "client.crt"))
           )
           context.key = OpenSSL::PKey::RSA.new(
-            File.read(File.join(certs_dir, 'client.key'))
+            File.read(File.join(certs_dir, "client.key"))
           )
           context
         end
       )
     end
 
-    it 'works via SSL' do
+    include_context "handles shared connections" do
+      let(:server) { dummy_ssl }
+    end
+
+    it "works via SSL" do
       response = client.get(dummy_ssl.endpoint)
-      expect(response.body.to_s).to eq('<!doctype html>')
+      expect(response.body.to_s).to eq("<!doctype html>")
     end
 
-    context 'with a mismatch host' do
-      it 'errors' do
-        expect { client.get(dummy_ssl.endpoint.gsub('127.0.0.1', 'localhost')) }
+    context "with a mismatch host" do
+      it "errors" do
+        expect { client.get(dummy_ssl.endpoint.gsub("127.0.0.1", "localhost")) }
           .to raise_error(OpenSSL::SSL::SSLError, /does not match/)
       end
     end
   end
 
-  describe '#perform' do
+  describe "#perform" do
     let(:client) { described_class.new }
 
-    it 'calls finish_response before actual performance' do
-      allow(TCPSocket).to receive(:open) { throw :halt }
-      expect(client).to receive(:finish_response)
-      catch(:halt) { client.head test_endpoint }
-    end
-
-    it 'calls finish_response once body was fully flushed' do
-      expect(client).to receive(:finish_response).twice.and_call_original
-      client.get(test_endpoint).to_s
+    it "calls finish_response once body was fully flushed" do
+      expect_any_instance_of(HTTP::Connection).to receive(:finish_response).and_call_original
+      client.get(dummy.endpoint).to_s
     end
 
-    context 'with HEAD request' do
-      it 'does not iterates through body' do
-        expect(client).to_not receive(:readpartial)
-        client.head(test_endpoint)
+    context "with HEAD request" do
+      it "does not iterates through body" do
+        expect_any_instance_of(HTTP::Connection).to_not receive(:readpartial)
+        client.head(dummy.endpoint)
       end
 
-      it 'finishes response after headers were received' do
-        expect(client).to receive(:finish_response).twice.and_call_original
-        client.head(test_endpoint)
+      it "finishes response after headers were received" do
+        expect_any_instance_of(HTTP::Connection).to receive(:finish_response).and_call_original
+        client.head(dummy.endpoint)
       end
     end
 
-    context 'when server closes connection unexpectedly' do
+    context "when server closes connection unexpectedly" do
       before do
         socket_spy = double
 
@@ -216,7 +244,7 @@ RSpec.describe HTTP::Client do
         allow(TCPSocket).to receive(:open) { socket_spy }
       end
 
-      context 'during headers reading' do
+      context "during headers reading" do
         let :chunks do
           [
             proc { "HTTP/1.1 200 OK\r\n" },
@@ -225,28 +253,28 @@ RSpec.describe HTTP::Client do
           ]
         end
 
-        it 'raises IOError' do
-          expect { client.get test_endpoint }.to raise_error IOError
+        it "raises IOError" do
+          expect { client.get dummy.endpoint }.to raise_error IOError
         end
       end
 
-      context 'after headers were flushed' do
+      context "after headers were flushed" do
         let :chunks do
           [
             proc { "HTTP/1.1 200 OK\r\n" },
             proc { "Content-Type: text/html\r\n\r\n" },
-            proc { 'unexpected end of f' },
+            proc { "unexpected end of f" },
             proc { fail EOFError }
           ]
         end
 
-        it 'reads partially arrived body' do
-          res = client.get(test_endpoint).to_s
-          expect(res).to eq 'unexpected end of f'
+        it "reads partially arrived body" do
+          res = client.get(dummy.endpoint).to_s
+          expect(res).to eq "unexpected end of f"
         end
       end
 
-      context 'when body and headers were flushed in one chunk' do
+      context "when body and headers were flushed in one chunk" do
         let :chunks do
           [
             proc { "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\nunexpected end of f" },
@@ -254,19 +282,19 @@ RSpec.describe HTTP::Client do
           ]
         end
 
-        it 'reads partially arrived body' do
-          res = client.get(test_endpoint).to_s
-          expect(res).to eq 'unexpected end of f'
+        it "reads partially arrived body" do
+          res = client.get(dummy.endpoint).to_s
+          expect(res).to eq "unexpected end of f"
         end
       end
     end
 
-    context 'when server fully flushes response in one chunk' do
+    context "when server fully flushes response in one chunk" do
       before do
         socket_spy = double
 
         chunks = [
-          <<-RESPONSE.gsub(/^\s*\| */, '').gsub(/\n/, "\r\n")
+          <<-RESPONSE.gsub(/^\s*\| */, "").gsub(/\n/, "\r\n")
           | HTTP/1.1 200 OK
           | Content-Type: text/html
           | Server: WEBrick/1.3.1 (Ruby/1.9.3/2013-11-22)
@@ -286,9 +314,9 @@ RSpec.describe HTTP::Client do
         allow(TCPSocket).to receive(:open) { socket_spy }
       end
 
-      it 'properly reads body' do
-        body = client.get(test_endpoint).to_s
-        expect(body).to eq '<!doctype html>'
+      it "properly reads body" do
+        body = client.get(dummy.endpoint).to_s
+        expect(body).to eq "<!doctype html>"
       end
     end
   end