manticore 0.3.0-java → 0.3.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0078d294015d5ca40a8e4afadc48973fb9246de1
-  data.tar.gz: bcff310d2a92c1e592cac51f63859a3ee483b63f
+  metadata.gz: 60f70fa96f93b6d9092b0455c139ae2aaf077fcf
+  data.tar.gz: ea9ab057b99806987c30ff3198cefd1d0d6ae455
 SHA512:
-  metadata.gz: ef8bfdbaf46d56810e0f137fb4cf96425a46bc22ecf157f9451093bcf9acf721ff4cff86161593ec7c95112a64793856e5345c19454569d0b3ca457c3216d99f
-  data.tar.gz: cd2bcc65d8d377d8622fae139e0f4c5e91816bcd0cea3926c8d1a5503eda2e68c28ef179cae5f153a389cd8c1474bb5ffbb3f789408e197f8d297b385b741e0a
+  metadata.gz: 24112c6208d50c27c68745340cf29c87f6cc37a8abeebb3bc051d2dc37ba883465f78e027e3dc4b5d77122c2bc9c2d9c2b6377607715bff5c953e38f0171ddb1
+  data.tar.gz: db3909740b474447d5467290992aad8b51fa34c39006714d14848650686f3ffaa095c3e8674df2f7ae0c834d7ac8fda4cc291485e790f9f3f44289a2041a647e

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## v0.3
-### v0.3.0 (pending)
+### v0.3.1
+* Added `automatic_retries` (default 3) parameter to client. The client will automatically retry requests that failed
+  due to socket exceptions and empty responses up to this number of times. The most practical effect of this setting is
+  to automatically retry when the pool reuses a connection that a client unexpectedly closed.
+* Added `request_timeout` to the RequestConfig used to construct requests.
+* Fixed implementation of the `:query` parameter for GET, HEAD, and DELETE requests.
+
+### v0.3.0
 
 * Major refactor of `Response`/`AsyncResponse` to eliminate redundant code. `AsyncResponse` has been removed and
   its functionality has been rolled into `Response`.

data/Gemfile

@@ -3,8 +3,8 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in manticore.gemspec
 gemspec
 
-gem "net-http-server"
-gem "rspec"
-gem "httpclient"
-gem "rack"
+gem "net-http-server", "~> 0.2"
+gem "rspec", "~> 2.14"
+gem "httpclient", "~> 2.3"
+gem "rack", "~> 1.5"
 gem "rake-compiler"

data/ext/manticore/org/manticore/Manticore.java

@@ -77,7 +77,7 @@ public class Manticore implements Library {
         byte[] tmp = new byte[4096];
         int l;
         while((l = instream.read(tmp)) != -1) {
-          block.call( context, RubyString.newString(context.getRuntime(), new ByteList(tmp, true), encoding) );
+          block.call( context, RubyString.newString(context.getRuntime(), new ByteList(tmp, 0, l, true), encoding) );
         }
       } finally {
         instream.close();

data/lib/manticore.rb

@@ -3,7 +3,7 @@ require 'uri'
 require 'cgi'
 require 'cgi/cookie'
 
-jars = ["httpcore-4.3.1", "httpclient-4.3.2-patched", "commons-logging-1.1.3", "commons-codec-1.6.jar"]
+jars = ["httpcore-4.3.1", "httpclient-4.3.2-patched", "commons-logging-1.1.3", "commons-codec-1.6.jar", "httpmime-4.3.2.jar"]
 jars.each do |jar|
   begin
     require_relative "./jar/#{jar}"

data/lib/manticore/client.rb

@@ -70,7 +70,9 @@ module Manticore
     include_package "org.apache.http.auth"
     include_package "java.util.concurrent"
     include_package "org.apache.http.client.protocol"
+    include_package 'org.apache.http.conn.ssl'
     java_import "org.apache.http.HttpHost"
+
     include ProxiesInterface
 
     # The default maximum pool size for requests
@@ -108,8 +110,10 @@ module Manticore
     # @option options [integer] request_timeout    (60)    Sets the timeout for requests. Raises {Manticore::Timeout} on failure.
     # @option options [integer] connect_timeout    (10)    Sets the timeout for connections. Raises Manticore::Timeout on failure.
     # @option options [integer] socket_timeout     (10)    Sets SO_TIMEOUT for open connections. A value of 0 is an infinite timeout. Raises Manticore::Timeout on failure.
+    # @option options [boolean] tcp_no_delay       (true)  Enable or disable Nagle's algorithm
     # @option options [integer] request_timeout    (60)    Sets the timeout for a given request. Raises Manticore::Timeout on failure.
     # @option options [integer] max_redirects      (5)     Sets the maximum number of redirects to follow.
+    # @option options [integer] automatic_retries  (3)     Sets the number of times the client will automatically retry failed requests.
     # @option options [boolean] expect_continue    (false) Enable support for HTTP 100
     # @option options [boolean] stale_check        (false) Enable support for stale connection checking. Adds overhead.
     # @option options [String]  proxy              Proxy host in form: http://proxy.org:1234
@@ -126,6 +130,20 @@ module Manticore
       builder.disable_content_compression if options.fetch(:compression, true) == false
       builder.set_proxy get_proxy_host(options[:proxy]) if options.key?(:proxy)
 
+      builder.set_retry_handler do |exception, executionCount, context|
+        if (executionCount > options.fetch(:automatic_retries, 3))
+          false
+        else
+          case exception
+          when Java::OrgApacheHttp::NoHttpResponseException, Java::JavaNet::SocketException
+            context.setAttribute "retryCount", executionCount
+            true
+          else
+            false
+          end
+        end
+      end
+
       # This should make it easier to reuse connections
       # TODO: Determine what this actually does!
       # builder.disable_connection_state
@@ -138,7 +156,8 @@ module Manticore
       end
 
       socket_config_builder = SocketConfig.custom
-      socket_config_builder.setSoTimeout( options.fetch(:socket_timeout, DEFAULT_SOCKET_TIMEOUT) * 1000 )
+      socket_config_builder.set_so_timeout( options.fetch(:socket_timeout, DEFAULT_SOCKET_TIMEOUT) * 1000 )
+      socket_config_builder.set_tcp_no_delay( options.fetch(:tcp_no_delay, true) )
       builder.set_default_socket_config socket_config_builder.build
 
       builder.set_connection_manager pool(options)
@@ -269,16 +288,24 @@ module Manticore
       HttpClientBuilder.create
     end
 
-    def pool_builder
-      PoolingHttpClientConnectionManager.new
+    def pool_builder(options)
+      if options.fetch(:ignore_ssl_validation, false)
+        context  = SSLContexts.custom.load_trust_material(nil, TrustSelfSignedStrategy.new).build
+        sslsf    = SSLConnectionSocketFactory.new(context, SSLConnectionSocketFactory::ALLOW_ALL_HOSTNAME_VERIFIER)
+        registry = RegistryBuilder.create.register("https", sslsf).build
+        PoolingHttpClientConnectionManager.new(registry)
+      else
+        PoolingHttpClientConnectionManager.new
+      end
     end
 
     def pool(options = {})
       @pool ||= begin
         @max_pool_size = options.fetch(:pool_max, DEFAULT_MAX_POOL_SIZE)
-        cm = pool_builder
+        cm = pool_builder options
         cm.set_default_max_per_route options.fetch(:pool_max_per_route, @max_pool_size)
         cm.set_max_total @max_pool_size
+
         Thread.new {
           loop {
             cm.closeExpiredConnections
@@ -332,12 +359,12 @@ module Manticore
     def uri_from_url_and_options(url, options)
       uri = Addressable::URI.parse url
       if options[:query]
-        uri.query_values ||= {}
+        v = uri.query_values || {}
         case options[:query]
         when Hash
-          uri.query_values.merge! options[:query]
+          uri.query_values = v.merge options[:query]
         when String
-          uri.query_values.merge! CGI.parse(options[:query])
+          uri.query_values = v.merge CGI.parse(options[:query])
         else
           raise "Queries must be hashes or strings"
         end
@@ -348,22 +375,25 @@ module Manticore
     def request_from_options(klass, url, options)
       req = klass.new uri_from_url_and_options(url, options).to_s
 
-      if ( options[:params] || options[:body] ) &&
+      if ( options[:params] || options[:body] || options[:entity]) &&
          ( req.instance_of?(HttpPost) || req.instance_of?(HttpPatch) || req.instance_of?(HttpPut) )
         if options[:params]
           req.set_entity hash_to_entity(options[:params])
         elsif options[:body]
           req.set_entity StringEntity.new(options[:body])
+        elsif options[:entity]
+          req.set_entity options[:entity]
         end
       end
 
       if options.key?(:proxy) || options.key?(:connect_timeout) || options.key?(:socket_timeout) || options.key?(:max_redirects) || options.key?(:follow_redirects)
         config = RequestConfig.custom()
-        config.set_proxy get_proxy_host(options[:proxy])          if options[:proxy]
-        config.set_connect_timeout options[:connect_timeout]      if options[:connect_timeout]
-        config.set_socket_timeout options[:socket_timeout]        if options[:socket_timeout]
-        config.set_max_redirects options[:max_redirects]          if options[:max_redirects]
-        config.set_redirects_enabled !!options[:follow_redirects] if options.fetch(:follow_redirects, nil) != nil
+        config.set_proxy get_proxy_host(options[:proxy])                if options[:proxy]
+        config.set_connect_timeout options[:connect_timeout]            if options[:connect_timeout]
+        config.set_socket_timeout options[:socket_timeout]              if options[:socket_timeout]
+        config.set_max_redirects options[:max_redirects]                if options[:max_redirects]
+        config.set_redirects_enabled !!options[:follow_redirects]       if options.fetch(:follow_redirects, nil) != nil
+        config.set_connection_request_timeout options[:request_timeout] if options[:request_timeout]
         req.set_config config.build
       end
 

data/lib/manticore/response.rb

@@ -47,11 +47,12 @@ module Manticore
         @client.execute @request, self, @context
         execute_complete
         return self
-      rescue Java::JavaNet::SocketTimeoutException, Java::OrgApacheHttpConn::ConnectTimeoutException, Java::OrgApacheHttp::NoHttpResponseException => e
+      rescue Java::JavaNet::SocketTimeoutException, Java::OrgApacheHttpConn::ConnectTimeoutException => e
         ex = Manticore::Timeout.new(e.get_cause)
       rescue Java::JavaNet::SocketException => e
         ex = Manticore::SocketException.new(e.get_cause)
-      rescue Java::OrgApacheHttpClient::ClientProtocolException, Java::JavaxNetSsl::SSLHandshakeException, Java::OrgApacheHttpConn::HttpHostConnectException => e
+      rescue Java::OrgApacheHttpClient::ClientProtocolException, Java::JavaxNetSsl::SSLHandshakeException, Java::OrgApacheHttpConn::HttpHostConnectException,
+             Java::OrgApacheHttp::NoHttpResponseException, Java::OrgApacheHttp::ConnectionClosedException => e
         ex = Manticore::ClientProtocolException.new(e.get_cause)
       rescue Java::JavaNet::UnknownHostException => e
         ex = Manticore::ResolutionFailure.new(e.get_cause)
@@ -194,6 +195,10 @@ module Manticore
     alias_method :completed,    :on_complete
     alias_method :on_completed, :on_complete
 
+    def times_retried
+      @context.get_attribute("retryCount") || 0
+    end
+
     private
 
     # Implementation of {http://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/client/ResponseHandler.html#handleResponse(org.apache.http.HttpResponse) ResponseHandler#handleResponse}

data/lib/manticore/version.rb

@@ -1,3 +1,3 @@
 module Manticore
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

data/spec/manticore/client_spec.rb

@@ -1,6 +1,10 @@
 require 'spec_helper'
 
+java_import 'org.apache.http.entity.mime.MultipartEntityBuilder'
+java_import 'org.apache.http.entity.ContentType'
+
 describe Manticore::Client do
+
   let(:client) { Manticore::Client.new }
 
   it "should fetch a URL and return a response" do
@@ -37,6 +41,24 @@ describe Manticore::Client do
     j["uri"]["port"].should == 55441
   end
 
+  describe "ignore_ssl_validation" do
+    context "when on" do
+      let(:client) { Manticore::Client.new ignore_ssl_validation: true }
+
+      it "should not break on SSL validation errors" do
+        expect { client.get("https://localhost:55444/").body }.to_not raise_exception
+      end
+    end
+
+    context "when off" do
+      let(:client) { Manticore::Client.new ignore_ssl_validation: false }
+
+      it "should break on SSL validation errors" do
+        expect { client.get("https://localhost:55444/").call }.to raise_exception(Manticore::ClientProtocolException)
+      end
+    end
+  end
+
   describe "lazy evaluation" do
     it "should not call synchronous requests by default" do
       req = client.get(local_server)
@@ -189,6 +211,13 @@ describe Manticore::Client do
       response = client.get(local_server)
       JSON.load(response.body)["method"].should == "GET"
     end
+
+    context "with a query" do
+      it "should work" do
+        response = client.get local_server, query: {foo: "bar"}
+        CGI.parse(JSON.load(response.body)["uri"]["query"])["foo"].should == ["bar"]
+      end
+    end
   end
 
   describe "#post" do
@@ -206,6 +235,13 @@ describe Manticore::Client do
       response = client.post(local_server, params: {key: "value"})
       JSON.load(response.body)["body"].should == "key=value"
     end
+
+    it "should send an arbitrary entity" do
+      f = open(__FILE__, "r").to_inputstream
+      multipart_entity = MultipartEntityBuilder.create.add_text_body("foo", "bar").add_binary_body("whatever", f , ContentType::TEXT_PLAIN, __FILE__)
+      response = client.post(local_server, entity: multipart_entity.build)
+      response.body.should match("should send an arbitrary entity")
+    end
   end
 
   describe "#put" do
@@ -258,14 +294,16 @@ describe Manticore::Client do
 
   describe "#execute!" do
     it "should perform multiple concurrent requests" do
-      @times = []
-      [55441, 55442].each do |port|
+      futures = [55441, 55442].map do |port|
         client.async.get("http://localhost:#{port}/?sleep=1").
-          on_success {|response| @times << Time.now.to_f }
+          on_success do |response|
+            Time.now.to_f
+          end
       end
 
       client.execute!
-      @times[0].should be_within(0.5).of(@times[1])
+      values = futures.map(&:callback_result)
+      (values[0] - values[1]).abs.should < 0.25
     end
 
     it "should return the results of the handler blocks" do
@@ -349,6 +387,74 @@ describe Manticore::Client do
     end
   end
 
+  context "with a misbehaving endpoint" do
+    before do
+      @socket = TCPServer.new 4567
+      @server = Thread.new do
+        puts "Accepting"
+        loop do
+          client = @socket.accept
+          client.puts([
+            "HTTP/1.1 200 OK",
+            "Keep-Alive: timeout=3000",
+            "Connection: Keep-Alive",
+            "Content-Length: 6",
+            "",
+            "Hello!"
+          ].join("\n"))
+          client.close
+        end
+      end
+    end
+
+    let(:client) { Manticore::Client.new keepalive: true, pool_max: 1 }
+
+    it "should retry 3 times by default" do
+      # The first time, reply with keepalive, then close the connection
+      # The second connection should succeed
+
+      request1 = client.get("http://localhost:4567/")
+      request2 = client.get("http://localhost:4567/")
+      expect { request1.call }.to_not raise_exception
+      expect { request2.call }.to_not raise_exception
+
+      request1.times_retried.should == 0
+      request2.times_retried.should == 1
+    end
+
+    context "when the max retry is restrictive" do
+      let(:client) { Manticore::Client.new keepalive: true, pool_max: 1, automatic_retries: 0 }
+
+      it "should retry 0 times and fail on the second request" do
+        # The first time, reply with keepalive, then close the connection
+        # The second connection should succeed
+        expect { client.get("http://localhost:4567/").call }.to_not raise_exception
+        expect { client.get("http://localhost:4567/").call }.to raise_exception(Manticore::SocketException)
+      end
+    end
+
+    context "when keepalive is off" do
+      let(:client) { Manticore::Client.new keepalive: false, pool_max: 1 }
+
+      it "should succeed without any retries" do
+        # The first time, reply with keepalive, then close the connection
+        # The second connection should succeed
+        request1 = client.get("http://localhost:4567/")
+        request2 = client.get("http://localhost:4567/")
+        expect { request1.call }.to_not raise_exception
+        expect { request2.call }.to_not raise_exception
+
+        request1.times_retried.should == 0
+        request2.times_retried.should == 0
+      end
+    end
+
+    after do
+      Thread.kill @server
+      @socket.close
+    end
+  end
+
   def get_connection(client, uri, &block)
     java_import "java.util.concurrent.TimeUnit"
     host = URI.parse(uri).host

data/spec/spec_helper.rb

@@ -4,6 +4,8 @@ require 'manticore'
 require 'zlib'
 require 'json'
 require 'rack'
+require 'webrick'
+require 'webrick/https'
 
 PORT = 55441
 
@@ -34,7 +36,7 @@ def start_server(port = PORT)
         sleep(query["sleep"].to_f)
       end
 
-      if cl = request[:headers]["Content-Length"]
+      if cl = request[:headers]["Content-Length"] || request[:headers]["Transfer-Encoding"] == "chunked"
         request[:body] = read_nonblock stream.socket
       end
 
@@ -46,6 +48,9 @@ def start_server(port = PORT)
         else
           [401, {'WWW-Authenticate' => 'Basic realm="test"'}, [""]]
         end
+      elsif request[:uri][:path] == "/failearly"
+        # Return an invalid HTTP response
+        []
       elsif match = request[:uri][:path].match(/\/cookies\/(\d)\/(\d)/)
         cookie_value = (request[:headers]["Cookie"] || "x=0").split("=").last.to_i
         if match[1].to_i == match[2].to_i
@@ -82,6 +87,21 @@ def stop_servers
   @servers.values.each(&:kill) if @servers
 end
 
+def start_ssl_server(port)
+  cert_name = [
+    %w[CN localhost],
+  ]
+  @servers[port] = Thread.new {
+    server = WEBrick::HTTPServer.new(:Port => port, :SSLEnable => true, :SSLCertName => cert_name, :Logger => WEBrick::Log.new("/dev/null"))
+    server.mount_proc "/" do |req, res|
+      res.body = "hello!"
+    end
+
+    server.start
+    puts "Server started?"
+  }
+end
+
 RSpec.configure do |c|
   require 'net/http/server'
 
@@ -89,6 +109,7 @@ RSpec.configure do |c|
     @server = {}
     start_server 55441
     start_server 55442
+    start_ssl_server 55444
   }
 
   c.after(:suite)  { stop_servers }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: manticore
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: java
 authors:
 - Chris Heald
@@ -30,7 +30,7 @@ cert_chain:
   E7PWS50D9moUJ6xWcemf0qKYC87qBFh0ng73awjG9uf+13lMslqJRMtek8C92cvh
   +R9zgQlbeNjy9O1i
   -----END CERTIFICATE-----
-date: 2014-03-28 00:00:00.000000000 Z
+date: 2014-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -95,6 +95,7 @@ files:
 - lib/jar/commons-logging-1.1.3.jar
 - lib/jar/httpclient-4.3.2-patched.jar
 - lib/jar/httpcore-4.3.1.jar
+- lib/jar/httpmime-4.3.2.jar
 - lib/jar/lazy_decompressing_stream.patch
 - lib/jar/manticore-ext.jar
 - lib/manticore.rb
@@ -133,7 +134,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.1
+rubygems_version: 2.2.2
 signing_key:
 specification_version: 4
 summary: Manticore is an HTTP client built on the Apache HttpCore components