http 0.8.0.pre3 → 0.8.0.pre4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5758483fce79ce2698462f1ffedc148d01d75c25
-  data.tar.gz: ac44a7864edf119f4287c9304537982887cdcdcd
+  metadata.gz: b7920fa65897a4adb435be3f24272554a59bb8f0
+  data.tar.gz: 9602b19d0e5f70eea417e69343a81159c356c3be
 SHA512:
-  metadata.gz: 17cf0d470610ce06db0ff695674ae80a59b1fb1b3a6403f1851aaf15284ecdba86e1584bc99ae9bcdb3fe431909111fe02e45f586afdaed0866a7a867cf481f3
-  data.tar.gz: 76101412bb754229a66b1612c3ccb656265abde996c643a1b7021f47ba05760ac85d98ff20f3c1f517a99da7f445101928e3fe168c8cce8b64a7d76ef418cdea
+  metadata.gz: e295f5c7d2b7c34ff6c4e0da1746361b777903d354910f0003eaded8c14cc180cd7e6e1edb11ce6980c63b851d0290c6e2dc0ba44c7bdee3cce036ee2ce17f3f
+  data.tar.gz: 7956cd926d6060cdc7186e1546bf8b33192f82cb71ce288c98b19f467b9e8192a4d3cfa92fd2a789e9c2dcf5631a89c7bda9136f359b8f1bb0e15a16fc14a9f0

data/CHANGES.md

@@ -1,4 +1,4 @@
-## 0.8.0.pre3 (2015-03-27)
+## 0.8.0.pre4 (2015-03-27)
 
 * Support for persistent HTTP connections (@zanker)
 * Add caching support. See #77 and #177. (@Asmod4n, @pezra)

data/lib/http.rb

@@ -1,6 +1,9 @@
 require "http/parser"
 
 require "http/errors"
+require "http/timeout/null"
+require "http/timeout/per_operation"
+require "http/timeout/global"
 require "http/chainable"
 require "http/client"
 require "http/connection"

data/lib/http/client.rb

@@ -17,6 +17,7 @@ module HTTP
 
     def initialize(default_options = {})
       @default_options = HTTP::Options.new(default_options)
+      @connection = nil
     end
 
     # Make an HTTP request

data/lib/http/connection.rb

@@ -16,14 +16,22 @@ module HTTP
 
       @parser = Response::Parser.new
 
-      @socket = options[:socket_class].open(req.socket_host, req.socket_port)
+      @socket = options[:timeout_class].new(options[:timeout_options])
+      @socket.connect(options[:socket_class], req.socket_host, req.socket_port)
 
-      start_tls(req.uri.host, options[:ssl_socket_class], options[:ssl_context]) if req.uri.is_a?(URI::HTTPS) && !req.using_proxy?
+      @socket.start_tls(
+        req.uri.host,
+        options[:ssl_socket_class],
+        options[:ssl_context]
+      ) if req.uri.is_a?(URI::HTTPS) && !req.using_proxy?
 
       reset_timer
     end
 
     # Send a request to the server
+    #
+    # @param [Request] Request to send to the server
+    # @return [Nil]
     def send_request(req)
       if pending_response
         fail StateError, "Tried to send a request while one is pending already. Make sure you read off the body."
@@ -60,7 +68,7 @@ module HTTP
       chunk.to_s
     end
 
-    # Reads data from socket up until headers
+    # Reads data from socket up until headers are loaded
     def read_headers!
       read_more BUFFER_SIZE until parser.headers
       set_keep_alive
@@ -131,23 +139,5 @@ module HTTP
     end
 
     private :read_more
-
-    # Starts the SSL connection
-    def start_tls(host, ssl_socket_class, ssl_context)
-      # TODO: abstract away SSLContexts so we can use other TLS libraries
-      ssl_context ||= OpenSSL::SSL::SSLContext.new
-      @socket = ssl_socket_class.new(socket, ssl_context)
-      socket.sync_close = true
-
-      socket.connect
-
-      if ssl_context.verify_mode == OpenSSL::SSL::VERIFY_PEER
-        socket.post_connection_check(host)
-      end
-
-      socket
-    end
-
-    private :start_tls
   end
 end

data/lib/http/errors.rb

@@ -11,6 +11,9 @@ module HTTP
   # Requested to do something when we're in the wrong state
   class StateError < ResponseError; end
 
+  # Generic Timeout error
+  class TimeoutError < Error; end
+
   # Generic Cache error
   class CacheError < Error; end
 

data/lib/http/options.rb

@@ -8,11 +8,13 @@ module HTTP
     @default_socket_class     = TCPSocket
     @default_ssl_socket_class = OpenSSL::SSL::SSLSocket
 
+    @default_timeout_class = HTTP::Timeout::Null
+
     @default_cache = Http::Cache::NullCache.new
 
     class << self
       attr_accessor :default_socket_class, :default_ssl_socket_class
-      attr_accessor :default_cache
+      attr_accessor :default_cache, :default_timeout_class
 
       def new(options = {})
         return options if options.is_a?(self)
@@ -41,6 +43,8 @@ module HTTP
     def initialize(options = {})
       defaults = {:response =>         :auto,
                   :proxy =>            {},
+                  :timeout_class =>    self.class.default_timeout_class,
+                  :timeout_options =>  {},
                   :socket_class =>     self.class.default_socket_class,
                   :ssl_socket_class => self.class.default_ssl_socket_class,
                   :cache =>            self.class.default_cache,
@@ -62,7 +66,7 @@ module HTTP
     %w(
       proxy params form json body follow response
       socket_class ssl_socket_class ssl_context
-      persistent keep_alive_timeout
+      persistent keep_alive_timeout timeout_class timeout_options
     ).each do |method_name|
       def_option method_name
     end

data/lib/http/timeout/global.rb

@@ -0,0 +1,99 @@
+# rubocop:disable Lint/HandleExceptions
+module HTTP
+  module Timeout
+    class Global < PerOperation
+      attr_reader :time_left, :total_timeout
+
+      def initialize(*args)
+        super
+
+        @time_left = connect_timeout + read_timeout + write_timeout
+        @total_timeout = time_left
+      end
+
+      # Abstracted out from the normal connect for SSL connections
+      def connect_with_timeout(*args)
+        reset_timer
+
+        begin
+          socket.connect_nonblock(*args)
+
+        rescue IO::WaitReadable
+          IO.select([socket], nil, nil, time_left)
+          log_time
+          retry
+
+        rescue Errno::EINPROGRESS
+          IO.select(nil, [socket], nil, time_left)
+          log_time
+          retry
+
+        rescue Errno::EISCONN
+        end
+      end
+
+      # Read from the socket
+      def readpartial(size)
+        reset_timer
+
+        begin
+          socket.read_nonblock(size)
+        rescue IO::WaitReadable
+          IO.select([socket], nil, nil, time_left)
+          log_time
+          retry
+        end
+      end
+
+      # Write to the socket
+      def write(data)
+        reset_timer
+
+        begin
+          socket << data
+        rescue IO::WaitWritable
+          IO.select(nil, [socket], nil, time_left)
+          log_time
+          retry
+        end
+      end
+
+      private
+
+      # Create a DNS resolver
+      def resolve_address(host)
+        addr = HostResolver.getaddress(host)
+        return addr if addr
+
+        reset_timer
+
+        addr = Resolv::DNS.open(:timeout => time_left) do |dns|
+          dns.getaddress
+        end
+
+        log_time
+
+        addr
+
+      rescue Resolv::ResolvTimeout
+        raise TimeoutError, "DNS timed out after #{total_timeout} seconds"
+      end
+
+      # Due to the run/retry nature of nonblocking I/O, it's easier to keep track of time
+      # via method calls instead of a block to monitor.
+      def reset_timer
+        @started = Time.now
+      end
+
+      def log_time
+        @time_left -= (Time.now - @started)
+        if time_left <= 0
+          fail TimeoutError, "Timed out after using the allocated #{total_timeout} seconds"
+        end
+
+        reset_timer
+      end
+    end
+  end
+end
+# rubocop:enable Lint/HandleExceptions

data/lib/http/timeout/null.rb

@@ -0,0 +1,51 @@
+require "forwardable"
+
+module HTTP
+  module Timeout
+    class Null
+      extend Forwardable
+
+      def_delegators :@socket, :close, :closed?
+
+      attr_reader :options, :socket
+
+      def initialize(options = {})
+        @options = options
+      end
+
+      # Connects to a socket
+      def connect(socket_class, host, port)
+        @socket = socket_class.open(host, port)
+      end
+
+      # Starts a SSL connection on a socket
+      def connect_ssl
+        socket.connect
+      end
+
+      # Configures the SSL connection and starts the connection
+      def start_tls(host, ssl_socket_class, ssl_context)
+        # TODO: abstract away SSLContexts so we can use other TLS libraries
+        ssl_context ||= OpenSSL::SSL::SSLContext.new
+        @socket = ssl_socket_class.new(socket, ssl_context)
+        socket.sync_close = true
+
+        connect_ssl
+
+        socket.post_connection_check(host) if ssl_context.verify_mode == OpenSSL::SSL::VERIFY_PEER
+      end
+
+      # Read from the socket
+      def readpartial(size)
+        socket.readpartial(size)
+      end
+
+      # Write to the socket
+      def write(data)
+        socket << data
+      end
+
+      alias_method :<<, :write
+    end
+  end
+end

data/lib/http/timeout/per_operation.rb

@@ -0,0 +1,117 @@
+# rubocop:disable Lint/HandleExceptions
+require "resolv"
+
+module HTTP
+  module Timeout
+    class PerOperation < Null
+      HostResolver = Resolv::Hosts.new.tap(&:lazy_initialize)
+
+      CONNECT_TIMEOUT = 0.25
+      WRITE_TIMEOUT = 0.25
+      READ_TIMEOUT = 0.25
+
+      attr_reader :read_timeout, :write_timeout, :connect_timeout
+
+      def initialize(*args)
+        super
+
+        @read_timeout = options.fetch(:read_timeout, READ_TIMEOUT)
+        @write_timeout = options.fetch(:write_timeout, WRITE_TIMEOUT)
+        @connect_timeout = options.fetch(:connect_timeout, CONNECT_TIMEOUT)
+      end
+
+      def connect(_, host, port)
+        # https://github.com/celluloid/celluloid-io/blob/master/lib/celluloid/io/tcp_socket.rb
+        begin
+          addr = Resolv::IPv4.create(host)
+        rescue ArgumentError
+        end
+
+        # Guess it's not IPv4! Is it IPv6?
+        begin
+          addr ||= Resolv::IPv6.create(host)
+        rescue ArgumentError
+        end
+
+        unless addr
+          addr = resolve_address(host)
+          fail Resolv::ResolvError, "DNS result has no information for #{host}" unless addr
+        end
+
+        case addr
+        when Resolv::IPv4
+          family = Socket::AF_INET
+        when Resolv::IPv6
+          family = Socket::AF_INET6
+        else fail ArgumentError, "unsupported address class: #{addr.class}"
+        end
+
+        @socket = Socket.new(family, Socket::SOCK_STREAM, 0)
+
+        connect_with_timeout(Socket.sockaddr_in(port, addr.to_s))
+      end
+
+      # No changes need to be made for the SSL connection
+      alias_method :connect_with_timeout, :connect_ssl
+
+      # Read data from the socket
+      def readpartial(size)
+        socket.read_nonblock(size)
+      rescue IO::WaitReadable
+        if IO.select([socket], nil, nil, read_timeout)
+          retry
+        else
+          raise TimeoutError, "Read timed out after #{read_timeout} seconds"
+        end
+      end
+
+      # Write data to the socket
+      def write(data)
+        socket.write_nonblock(data)
+      rescue IO::WaitWritable
+        if IO.select(nil, [socket], nil, write_timeout)
+          retry
+        else
+          raise TimeoutError, "Read timed out after #{write_timeout} seconds"
+        end
+      end
+
+      private
+
+      # Actually do the connect after we're setup
+      def connect_with_timeout(*args)
+        socket.connect_nonblock(*args)
+
+      rescue IO::WaitReadable
+        if IO.select([socket], nil, nil, connect_timeout)
+          retry
+        else
+          raise TimeoutError, "Connection timed out after #{connect_timeout} seconds"
+        end
+
+      rescue Errno::EINPROGRESS
+        if IO.select(nil, [socket], nil, connect_timeout)
+          retry
+        else
+          raise TimeoutError, "Connection timed out after #{connect_timeout} seconds"
+        end
+
+      rescue Errno::EISCONN
+      end
+
+      # Create a DNS resolver
+      def resolve_address(host)
+        addr = HostResolver.getaddress(host)
+        return addr if addr
+
+        Resolv::DNS.open(:timeout => connect_timeout) do |dns|
+          dns.getaddress
+        end
+
+      rescue Resolv::ResolvTimeout
+        raise TimeoutError, "DNS timed out after #{connect_timeout} seconds"
+      end
+    end
+  end
+end
+# rubocop:enable Lint/HandleExceptions

data/lib/http/version.rb

@@ -1,3 +1,3 @@
 module HTTP
-  VERSION = "0.8.0.pre3"
+  VERSION = "0.8.0.pre4"
 end

data/spec/lib/http/client_spec.rb

@@ -1,4 +1,4 @@
-require "support/connection_reuse_shared"
+require "support/http_handling_shared"
 require "support/dummy_server"
 require "http/cache"
 
@@ -167,44 +167,34 @@ RSpec.describe HTTP::Client do
     end
   end
 
-  include_context "handles shared connections" do
-    let(:reuse_conn) { nil }
-    let(:keep_alive_timeout) { 5 }
-
+  include_context "HTTP handling" do
+    let(:options) { {} }
     let(:server) { dummy }
-    let(:client) do
-      described_class.new(
-        :persistent => reuse_conn,
-        :keep_alive_timeout => keep_alive_timeout
-      )
-    end
+    let(:client) { described_class.new(options) }
   end
 
   describe "SSL" do
-    let(:reuse_conn) { nil }
-    let(:keep_alive_timeout) { 5 }
-
     let(:client) do
       described_class.new(
-        :persistent => reuse_conn,
-        :keep_alive_timeout => keep_alive_timeout,
-        :ssl_context => OpenSSL::SSL::SSLContext.new.tap do |context|
-          context.options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
-
-          context.verify_mode = OpenSSL::SSL::VERIFY_PEER
-          context.ca_file = File.join(certs_dir, "ca.crt")
-          context.cert = OpenSSL::X509::Certificate.new(
-            File.read(File.join(certs_dir, "client.crt"))
-          )
-          context.key = OpenSSL::PKey::RSA.new(
-            File.read(File.join(certs_dir, "client.key"))
-          )
-          context
-        end
+        options.merge(
+          :ssl_context => OpenSSL::SSL::SSLContext.new.tap do |context|
+            context.options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
+
+            context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+            context.ca_file = File.join(certs_dir, "ca.crt")
+            context.cert = OpenSSL::X509::Certificate.new(
+              File.read(File.join(certs_dir, "client.crt"))
+            )
+            context.key = OpenSSL::PKey::RSA.new(
+              File.read(File.join(certs_dir, "client.key"))
+            )
+            context
+          end
+        )
       )
     end
 
-    include_context "handles shared connections" do
+    include_context "HTTP handling", true do
       let(:server) { dummy_ssl }
     end
 

data/spec/lib/http/options/merge_spec.rb

@@ -34,10 +34,13 @@ RSpec.describe HTTP::Options, "merge" do
       :json      => {:bar => "bar"},
       :keep_alive_timeout => 10,
       :headers   => {:accept  => "xml", :bar => "bar"},
+      :timeout_options => {:foo => :bar},
       :proxy     => {:proxy_address => "127.0.0.1", :proxy_port => 8080})
 
     expect(foo.merge(bar).to_hash).to eq(
       :response  => :parsed_body,
+      :timeout_class  => described_class.default_timeout_class,
+      :timeout_options => {:foo => :bar},
       :params    => {:plop => "plip"},
       :form      => {:bar => "bar"},
       :body      => "body-bar",

data/spec/support/dummy_server/servlet.rb

@@ -40,6 +40,20 @@ class DummyServer < WEBrick::HTTPServer
       end
     end
 
+    get "/sleep" do |_, res|
+      sleep 2
+
+      res.status = 200
+      res.body   = "hello"
+    end
+
+    post "/sleep" do |_, res|
+      sleep 2
+
+      res.status = 200
+      res.body   = "hello"
+    end
+
     ["", "/1", "/2"].each do |path|
       get "/socket#{path}" do |req, res|
         self.class.sockets << req.instance_variable_get(:@socket)

data/spec/support/http_handling_shared.rb

@@ -0,0 +1,207 @@
+RSpec.shared_context "HTTP handling" do |ssl = false|
+  describe "timeouts" do
+    let(:conn_timeout) { 1 }
+    let(:read_timeout) { 1 }
+    let(:write_timeout) { 1 }
+
+    let(:options) do
+      {
+        :timeout_class => timeout_class,
+        :timeout_options => {
+          :connect_timeout => conn_timeout,
+          :read_timeout => read_timeout,
+          :write_timeout => write_timeout
+        }
+      }
+    end
+
+    context "without timeouts" do
+      let(:timeout_class) { HTTP::Timeout::Null }
+      let(:conn_timeout) { 0 }
+      let(:read_timeout) { 0 }
+      let(:write_timeout) { 0 }
+
+      it "works" do
+        expect(client.get(server.endpoint).body.to_s).to eq("<!doctype html>")
+      end
+    end
+
+    context "with a per operation timeout" do
+      let(:timeout_class) { HTTP::Timeout::PerOperation }
+
+      let(:response) { client.get(server.endpoint).body.to_s }
+
+      it "works" do
+        expect(response).to eq("<!doctype html>")
+      end
+
+      context "connection" do
+        context "of 1" do
+          let(:conn_timeout) { 1 }
+
+          it "does not time out" do
+            expect { response }.to_not raise_error
+          end
+        end
+      end
+
+      context "read" do
+        context "of 0" do
+          let(:read_timeout) { 0 }
+
+          it "times out" do
+            expect { response }.to raise_error(HTTP::TimeoutError, /Read/i)
+          end
+        end
+
+        context "of 2.5" do
+          let(:read_timeout) { 2.5 }
+
+          it "does not time out" do
+            expect { client.get("#{server.endpoint}/sleep").body.to_s }.to_not raise_error
+          end
+        end
+      end
+    end
+
+    context "with a global timeout" do
+      let(:timeout_class) { HTTP::Timeout::Global }
+
+      let(:conn_timeout) { 0 }
+      let(:read_timeout) { 1 }
+      let(:write_timeout) { 0 }
+
+      let(:response) { client.get(server.endpoint).body.to_s }
+
+      context "with localhost" do
+        let(:endpoint) { server.endpoint.sub("127.0.0.1", "localhost") }
+
+        it "errors if DNS takes too long" do
+          # Block the localhost lookup
+          expect(timeout_class::HostResolver)
+            .to receive(:getaddress).with("localhost").and_return(nil)
+
+          # Request
+          expect(Resolv::DNS).to receive(:open).with(:timeout => 1) do |_|
+            sleep 1.25
+            "127.0.0.1"
+          end
+
+          expect { client.get(server.endpoint.sub("127.0.0.1", "localhost")) }
+            .to raise_error(HTTP::TimeoutError, /Timed out/)
+        end
+      end
+
+      it "errors if connecting takes too long" do
+        socket = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
+
+        fake_socket = double(:to_io => socket)
+        expect(fake_socket).to receive(:connect_nonblock) do |*args|
+          sleep 1.25
+          socket.connect_nonblock(*args)
+        end
+
+        allow_any_instance_of(timeout_class).to receive(:socket).and_return(fake_socket)
+
+        expect { response }.to raise_error(HTTP::TimeoutError, /Timed out/)
+      end
+
+      it "errors if reading takes too long" do
+        expect { client.get("#{server.endpoint}/sleep").body.to_s }
+          .to raise_error(HTTP::TimeoutError, /Timed out/)
+      end
+
+      unless ssl
+        it "errors if writing takes too long" do
+          socket = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
+          allow_any_instance_of(timeout_class).to receive(:socket).and_return(socket)
+
+          expect(socket).to receive(:<<) do |*|
+            sleep 1.25
+          end
+
+          expect { response }.to raise_error(HTTP::TimeoutError, /Timed out/)
+        end
+      end
+    end
+  end
+
+  describe "connection reuse" do
+    let(:sockets_used) do
+      [
+        client.get("#{server.endpoint}/socket/1").body.to_s,
+        client.get("#{server.endpoint}/socket/2").body.to_s
+      ]
+    end
+
+    context "when enabled" do
+      let(:options) { {:persistent => server.endpoint} }
+
+      context "without a host" do
+        it "infers host from persistent config" do
+          expect(client.get("/").body.to_s).to eq("<!doctype html>")
+        end
+      end
+
+      it "re-uses the socket" do
+        expect(sockets_used).to_not include("")
+        expect(sockets_used.uniq.length).to eq(1)
+      end
+
+      context "when trying to read a stale body" do
+        it "errors" do
+          client.get("#{server.endpoint}/not-found")
+          expect { client.get(server.endpoint) }.to raise_error(HTTP::StateError, /Tried to send a request/)
+        end
+      end
+
+      context "when reading a cached body" do
+        it "succeeds" do
+          first_res = client.get(server.endpoint)
+          first_res.body.to_s
+
+          second_res = client.get(server.endpoint)
+
+          expect(first_res.body.to_s).to eq("<!doctype html>")
+          expect(second_res.body.to_s).to eq("<!doctype html>")
+        end
+      end
+
+      context "with a socket issue" do
+        it "transparently reopens" do
+          first_socket = client.get("#{server.endpoint}/socket").body.to_s
+          expect(first_socket).to_not eq("")
+          # Kill off the sockets we used
+          # rubocop:disable Style/RescueModifier
+          DummyServer::Servlet.sockets.each do |socket|
+            socket.close rescue nil
+          end
+          DummyServer::Servlet.sockets.clear
+          # rubocop:enable Style/RescueModifier
+
+          # Should error because we tried to use a bad socket
+          expect { client.get("#{server.endpoint}/socket").body.to_s }.to raise_error(IOError)
+
+          # Should succeed since we create a new socket
+          second_socket = client.get("#{server.endpoint}/socket").body.to_s
+          expect(second_socket).to_not eq(first_socket)
+        end
+      end
+
+      context "with a change in host" do
+        it "errors" do
+          expect { client.get("https://invalid.com/socket") }.to raise_error(/Persistence is enabled/i)
+        end
+      end
+    end
+
+    context "when disabled" do
+      let(:options) { {} }
+
+      it "opens new sockets" do
+        expect(sockets_used).to_not include("")
+        expect(sockets_used.uniq.length).to eq(2)
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: http
 version: !ruby/object:Gem::Version
-  version: 0.8.0.pre3
+  version: 0.8.0.pre4
 platform: ruby
 authors:
 - Tony Arcieri
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-28 00:00:00.000000000 Z
+date: 2015-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
@@ -104,6 +104,9 @@ files:
 - lib/http/response/status.rb
 - lib/http/response/status/reasons.rb
 - lib/http/response/string_body.rb
+- lib/http/timeout/global.rb
+- lib/http/timeout/null.rb
+- lib/http/timeout/per_operation.rb
 - lib/http/version.rb
 - logo.png
 - spec/lib/http/cache/headers_spec.rb
@@ -138,6 +141,7 @@ files:
 - spec/support/create_certs.rb
 - spec/support/dummy_server.rb
 - spec/support/dummy_server/servlet.rb
+- spec/support/http_handling_shared.rb
 - spec/support/proxy_server.rb
 - spec/support/servers/config.rb
 - spec/support/servers/runner.rb
@@ -198,6 +202,7 @@ test_files:
 - spec/support/create_certs.rb
 - spec/support/dummy_server.rb
 - spec/support/dummy_server/servlet.rb
+- spec/support/http_handling_shared.rb
 - spec/support/proxy_server.rb
 - spec/support/servers/config.rb
 - spec/support/servers/runner.rb