manticore 0.7.1-java → 0.9.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 67e67c0611ad51fbaf279f456ef5a9e463038f1017c0eb8c7665032a05534b59
-  data.tar.gz: 0ecf2a3014befbbc5b26d95c3ee36809ccc8605e450631e4ec0f1c143225bbc5
+  metadata.gz: 381fa920f4a6ca174342719444ebdf44d3f1b23694840cbe942e2806094656b4
+  data.tar.gz: cb27a429bd009ab358c28beb8e77143fccb384076f1a03fcfd325e9d8055576b
 SHA512:
-  metadata.gz: 19f7db686cff19817b8310f7838f9115d343ab7516e7cbab066c0f0f7d223c7a0a42c40f9e361426c07345088dbdb818cabab59fb2a481d35050f939e27afafd
-  data.tar.gz: ebc35118980baac6eff3e1f795f5ad3aa0af01c7ac29cd6a9502f9300fb1da88d8b97ecfc7c1198d5902a35ab2501179df775791e3306e27004e85d690145bc2
+  metadata.gz: dc3b33be3c5fe29ba240859cfd25455a3d30b45285414773426e9780e5eab016ca7373fdb2d7fed368af00781e502650a1c75b03f577ca316206ef8c1abc1bd3
+  data.tar.gz: f983c8e4b6cf27bb91909d0cd83227d684498ac4a6ca72d3249a54cc43a4e5a5051069ba32ee40d7bc9028792fb07b70472907b7214cd4a2ab6a858e4d648bd8

data/.travis.yml

@@ -4,18 +4,22 @@ cache:
   - bundler
   - directories:
     - $HOME/.m2
-rvm:
-  - jruby-9.2.16.0 # Ruby 2.5
-jdk:
-  - oraclejdk8
-  - openjdk8
-  - openjdk11
 before_install:
   - gem install bundler -v 1.17.3
 matrix:
   include:
     - rvm: jruby-head
       jdk: openjdk11
+    - rvm: jruby-9.3.4.0 # Ruby 2.6
+      jdk: openjdk11
+    - rvm: jruby-9.3.4.0
+      jdk: openjdk11
+      env:
+        - JRUBY_OPTS="-Xcompile.invokedynamic -Xjit.threshold=0"
+    - rvm: jruby-9.2.20.0 # Ruby 2.5
+      jdk: oraclejdk8
+    - rvm: jruby-9.2.20.0
+      jdk: openjdk11
     - rvm: jruby-9.1.17.0 # Ruby 2.3
       jdk: openjdk8
   allow_failures:

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+### v0.9.1
+
+* [fix] work-around JRuby 9.3.4 compatibility
+* [refactor] delay closing async request queue on client.close
+
+### v0.9.0
+
+* [feat] revamped client.close to release resources (#108)
+* [fix] null password handling when loading keystore
+* [fix] ambiguous Java method selection
+* [refactor] revisit hostname verification (#103) 
+* [feat] ssl[:trust_strategy]: wrapper for `org.apache.http.conn.ssl.TrustStrategy` (#106)
+
+### v0.8.0
+
+* [feat] restore compat with (legacy) verify: false (#102)
+* Accept untrusted certs when SSL verify is disabled (#100)
+* [deps] update http-client to 4.5.13 (#99)
+
 ## v0.7
 
 ### v0.7.1

data/Gemfile

@@ -4,13 +4,15 @@ source 'https://rubygems.org'
 gemspec
 
 group :development, :test do
-  gem "net-http-server", "~> 0.2"
+  gem "rake-compiler", require: false
+  gem "simplecov"
+
   gem "rspec", "~> 3.0"
   gem "rspec-its"
-  gem "httpclient", "~> 2.3"
-  gem "rack", ">= 2.1.4"
-  gem "rake-compiler"
-  gem "gserver"
-  gem "simplecov"
-  gem "json"
+
+  gem "rack", ">= 2.1.4", require: false
+  gem "json", require: false
+  gem "webrick", require: false
+  gem "net-http-server", require: false
+  gem "gserver", require: false
 end

data/README.md

@@ -2,7 +2,7 @@
 
 **Note**: While I'll continue to maintain the library here, I've moved the canonical copy to Gitlab at https://gitlab.com/cheald/manticore - it is preferred that you submit issues and PRs there.
 
-[![Build Status](https://travis-ci.org/cheald/manticore.svg?branch=master)](https://travis-ci.org/cheald/manticore)
+[![Build Status](https://app.travis-ci.com/cheald/manticore.svg?branch=master)](https://app.travis-ci.com/cheald/manticore)
 
 Manticore is a fast, robust HTTP client built on the Apache HTTPClient libraries. It is only compatible with JRuby.
 

data/Rakefile

@@ -36,6 +36,7 @@ task :generate_certs do
     # Create the CA
     "#{openssl} genrsa 4096 | #{openssl} pkcs8 -topk8 -nocrypt -out #{root}/root-ca.key",
     "#{openssl} req -sha256 -x509 -newkey rsa:4096 -nodes -key #{root}/root-ca.key -sha256 -days 365 -out #{root}/root-ca.crt -subj \"/C=US/ST=The Internet/L=The Internet/O=Manticore CA/OU=Manticore/CN=localhost\"",
+    "#{openssl} req -sha256 -x509 -newkey rsa:4096 -nodes -key #{root}/root-ca.key -sha256 -days 365 -out #{root}/root-untrusted-ca.crt -subj \"/C=US/ST=The Darknet/L=The Darknet/O=Manticore CA/OU=Manticore/CN=localhost\"",
 
     # Create the client CSR, key, and signed cert
     "#{openssl} genrsa 4096 | #{openssl} pkcs8 -topk8 -nocrypt -out #{root}/client.key",
@@ -48,6 +49,7 @@ task :generate_certs do
     "#{openssl} req -sha256 -key #{root}/host.key -newkey rsa:4096 -out #{root}/host.csr -subj \"/C=US/ST=The Internet/L=The Internet/O=Manticore Host/OU=Manticore/CN=localhost\"",
     "#{openssl} x509 -req -in #{root}/host.csr -CA #{root}/root-ca.crt -CAkey #{root}/root-ca.key -CAcreateserial -out #{root}/host.crt -sha256 -days 1",
     "#{openssl} x509 -req -in #{root}/host.csr -CA #{root}/root-ca.crt -CAkey #{root}/root-ca.key -CAcreateserial -out #{root}/host-expired.crt -sha256 -days -7",
+    "#{openssl} x509 -req -in #{root}/host.csr -CA #{root}/root-untrusted-ca.crt -CAkey #{root}/root-ca.key -CAcreateserial -out #{root}/host-untrusted.crt -sha256 -days 1",
 
     "#{keytool} -import -file #{root}/root-ca.crt -alias rootCA -keystore #{root}/truststore.jks -noprompt -storepass test123",
     "#{openssl} pkcs12 -export -clcerts -out #{root}/client.p12 -inkey #{root}/client.key -in #{root}/client.crt -certfile #{root}/root-ca.crt -password pass:test123",

data/lib/manticore/client/trust_strategies.rb

@@ -0,0 +1,119 @@
+module Manticore
+  class Client
+    ##
+    # TrustStrategies is a utility module that provides helpers for
+    # working with org.apache.http.conn.ssl.TrustStrategy
+    module TrustStrategies
+      # Coerces to org.apache.http.conn.ssl.TrustStrategy, allowing nil pass-through
+      #
+      # @overload coerce(coercible)
+      #   @param coercible [nil|TrustStrategy]
+      #   @return [nil,TrustStrategy]
+      # @overload coerce(coercible)
+      #   @param coercible [Proc<(Array<OpenSSL::X509::Certificate>,String)>:Boolean]
+      #     A proc that accepts two arguments and returns a boolean value, and is effectively a
+      #     Ruby-native implementation of `org.apache.http.conn.ssl.TrustStrategy#isTrusted`.
+      #       @param cert_chain [Enumerable<OpenSSL::X509::Certificate>]: the peer's certificate chain
+      #       @param auth_type [String]: the authentication type based on the client certificate
+      #       @raise [OpenSSL::X509::CertificateError]: thrown if the certificate is not trusted or invalid
+      #       @return [Boolean]: true if the certificate can be trusted without verification by the trust manager,
+      #                          false otherwise.
+      #   @example: CA Trusted Fingerprint
+      #      ca_trusted_fingerprint = lambda do |cert_chain, type|
+      #        cert_chain.lazy
+      #                  .map(&:to_der)
+      #                  .map(&::Digest::SHA256.method(:hexdigest))
+      #                  .include?("324a87eebb19265ffb675dc345eb0f3b5d9de3f015159227a00fe552291d4cc4")
+      #      end
+      #      TrustStrategies.coerce(ca_trusted_fingerprint)
+      def self.coerce(coercible)
+        case coercible
+        when org.apache.http.conn.ssl.TrustStrategy, nil then coercible
+        when ::Proc                                      then CustomTrustStrategy.new(coercible)
+        else fail(ArgumentError, "No implicit conversion of #{coercible} to #{self}")
+        end
+      end
+
+      # Combines two possibly-nil TrustStrategies-coercible objects into a
+      # single org.apache.http.conn.ssl.TrustStrategy, or to nil if both are nil.
+      #
+      # @param lhs [nil|TrustStrategie#coerce]
+      # @param rhs [nil|TrustStrategies#coerce]
+      # @return [nil,org.apache.http.conn.ssl.TrustStrategy]
+      def self.combine(lhs, rhs)
+        return coerce(rhs) if lhs.nil?
+        return coerce(lhs) if rhs.nil?
+
+        CombinedTrustStrategy.new(coerce(lhs), coerce(rhs))
+      end
+    end
+
+    ##
+    # @api private
+    # A CombinedTrustStrategy can be used to bypass the Trust Manager if
+    # *EITHER* TrustStrategy trusts the provided certificate chain.
+    # @see TrustStrategies::combine
+    class CombinedTrustStrategy
+      include org.apache.http.conn.ssl.TrustStrategy
+
+      ##
+      # @api private
+      # @see TrustStrategies::combine
+      def initialize(lhs, rhs)
+        @lhs = lhs
+        @rhs = rhs
+        super()
+      end
+
+      ##
+      # @override (see org.apache.http.conn.ssl.TrustStrategy#isTrusted)
+      def trusted?(chain, type)
+        @lhs.trusted?(chain, type) || @rhs.trusted?(chain, type)
+      end
+    end
+
+
+    ##
+    # @api private
+    # A CustomTrustStrategy is an org.apache.http.conn.ssl.TrustStrategy
+    # defined with a proc that uses Ruby OpenSSL::X509::Certificates
+    # @see TrustStrategies::coerce(Proc)
+    class CustomTrustStrategy
+      include org.apache.http.conn.ssl.TrustStrategy
+
+      ##
+      # @see TrustStrategies.coerce(Proc)
+      def initialize(proc)
+        fail(ArgumentError, "2-arity proc required") unless proc.arity == 2
+        @trust_strategy = proc
+      end
+
+      CONVERT_JAVA_CERTIFICATE_TO_RUBY = -> (java_cert) { ::OpenSSL::X509::Certificate.new(java_cert.encoded) }
+      private_constant :CONVERT_JAVA_CERTIFICATE_TO_RUBY
+
+      ##
+      # @override (see org.apache.http.conn.ssl.TrustStrategy#isTrusted)
+      def trusted?(java_chain, type)
+        @trust_strategy.call(java_chain.lazy.map(&CONVERT_JAVA_CERTIFICATE_TO_RUBY), String.new(type))
+      rescue OpenSSL::X509::CertificateError => e
+        raise java_certificate_exception(e)
+      end
+
+      private
+
+      begin
+        # Ruby exceptions can be converted to Throwable since JRuby 9.2
+        Exception.new("sentinel").to_java(java.lang.Throwable)
+        def java_certificate_exception(ruby_certificate_error)
+          throwable = ruby_certificate_error.to_java(java.lang.Throwable)
+          java.security.cert.CertificateException.new(throwable)
+        end
+      rescue TypeError
+        def java_certificate_exception(ruby_certificate_error)
+          message = ruby_certificate_error.message
+          java.security.cert.CertificateException.new(message)
+        end
+      end
+    end
+  end
+end

data/lib/manticore/client.rb

@@ -69,7 +69,6 @@ module Manticore
     include_package "org.apache.http.client.config"
     include_package "org.apache.http.config"
     include_package "org.apache.http.conn.socket"
-    include_package "org.apache.http.impl.client"
     include_package "org.apache.http.impl.conn"
     include_package "org.apache.http.entity"
     include_package "org.apache.http.message"
@@ -86,24 +85,43 @@ module Manticore
     java_import "org.manticore.HttpGetWithEntity"
     java_import "org.manticore.HttpDeleteWithEntity"
     java_import "org.apache.http.auth.UsernamePasswordCredentials"
+    java_import "org.apache.http.conn.ssl.DefaultHostnameVerifier"
+    java_import "org.apache.http.conn.ssl.NoopHostnameVerifier"
     java_import "org.apache.http.conn.ssl.SSLConnectionSocketFactory"
-    java_import "org.apache.http.conn.ssl.SSLContextBuilder"
+    java_import "org.apache.http.conn.ssl.TrustAllStrategy"
     java_import "org.apache.http.conn.ssl.TrustSelfSignedStrategy"
     java_import "org.apache.http.client.utils.URIBuilder"
     java_import "org.apache.http.impl.DefaultConnectionReuseStrategy"
+    java_import "org.apache.http.impl.NoConnectionReuseStrategy"
     java_import "org.apache.http.impl.auth.BasicScheme"
+    java_import "org.apache.http.impl.client.BasicAuthCache"
+    java_import "org.apache.http.impl.client.BasicCookieStore"
+    java_import "org.apache.http.impl.client.BasicCredentialsProvider"
+    java_import "org.apache.http.impl.client.HttpClientBuilder"
+    java_import "org.apache.http.ssl.SSLContextBuilder"
+    java_import "org.apache.http.ssl.TrustStrategy"
 
     # This is a class rather than a proc because the proc holds a closure around
     # the instance of the Client that creates it.
     class ExecutorThreadFactory
-      include ::Java::JavaUtilConcurrent::ThreadFactory
+      include java.util.concurrent.ThreadFactory
+
+      java_import 'java.lang.Thread'
+
+      @@factory_no = java.util.concurrent.atomic.AtomicInteger.new
+
+      def initialize(client)
+        @thread_no = java.util.concurrent.atomic.AtomicInteger.new
+        @name_prefix = "manticore##{client.object_id}-#{@@factory_no.increment_and_get}-"
+      end
 
       def newThread(runnable)
-        thread = Executors.defaultThreadFactory.newThread(runnable)
-        thread.daemon = true
-        return thread
+        thread = Thread.new(runnable, @name_prefix + @thread_no.increment_and_get.to_s)
+        thread.setDaemon(true)
+        thread
       end
     end
+    private_constant :ExecutorThreadFactory
 
     include ProxiesInterface
 
@@ -161,10 +179,12 @@ module Manticore
     # @option options [Hash]            ssl                                        Hash of options for configuring SSL
     # @option options [Array<String>]   ssl[:protocols]            (nil)       A list of protocols that Manticore should accept
     # @option options [Array<String>]   ssl[:cipher_suites]        (nil)       A list of cipher suites that Manticore should accept
-    # @option options [Symbol]          ssl[:verify]               (:strict)   Hostname verification setting. Set to `:disable` to turn off hostname verification. Setting to `:browser` will
+    # @option options [Symbol]          ssl[:verify]               (:default)  Hostname verification setting. Set to `:none` to turn off hostname verification. Setting to `:browser` will
     #                                                                            cause Manticore to accept a certificate for *.foo.com for all subdomains and sub-subdomains (eg a.b.foo.com).
-    #                                                                            The default `:strict` is like `:browser` except it'll only accept a single level of subdomains for wildcards,
+    #                                                                            The default `:default` is like `:browser` but more strict - only accepts a single level of subdomains for wildcards,
     #                                                                            eg `b.foo.com` will be accepted for a `*.foo.com` certificate, but `a.b.foo.com` will not be.
+    # @option options [Client::TrustStrategiesInterface] ssl[:trust_strategy] (nil)     A trust strategy to use in addition to any built by `ssl[:verify]`.
+    #                                                                                                @see Client::TrustStrategiesInterface#coerce
     # @option options [String]          ssl[:truststore]          (nil)        Path to a custom trust store to use the verifying SSL connections
     # @option options [String]          ssl[:truststore_password] (nil)        Password used for decrypting the server trust store
     # @option options [String]          ssl[:truststore_type]     (nil)        Format of the trust store, ie "JKS" or "PKCS12". If left nil, the type will be inferred from the truststore filename.
@@ -177,8 +197,8 @@ module Manticore
     # @option options [boolean]         ssl[:track_state]         (false)      Turn on or off connection state tracking. This helps prevent SSL information from leaking across threads, but means that connections
     #                                                                             can't be shared across those threads. This should generally be left off unless you know what you're doing.
     def initialize(options = {})
-      @finalizers = []
-      self.class.shutdown_on_finalize self, @finalizers
+      @finalizer = Finalizer.new
+      self.class.shutdown_on_finalize self, @finalizer
 
       builder = client_builder
       builder.set_user_agent options.fetch(:user_agent, "Manticore #{VERSION}")
@@ -198,7 +218,7 @@ module Manticore
 
       @keepalive = options.fetch(:keepalive, true)
       if @keepalive == false
-        builder.set_connection_reuse_strategy { |response, context| false }
+        builder.set_connection_reuse_strategy NoConnectionReuseStrategy::INSTANCE # return false
       else
         builder.set_connection_reuse_strategy DefaultConnectionReuseStrategy.new
       end
@@ -344,21 +364,58 @@ module Manticore
       end
     end
 
-    # Free resources associated with the CloseableHttpClient
-    def close
-      @client.close if @client
+    # Release resources held by this client, namely:
+    #  - close the internal http client
+    #  - shutdown the connection pool
+    #  - stops accepting async requests in the executor
+    #
+    # After this call the client is no longer usable.
+    # @note In versions before 0.9 this method only closed the underlying `CloseableHttpClient`
+    def close(await: nil)
+      ObjectSpace.undefine_finalizer(self)
+      @finalizer.call # which does ~ :
+      # @executor&.shutdown rescue nil
+      # @client&.close
+      # @pool&.shutdown rescue nil
+      case await
+      when false, nil
+        @executor&.shutdown_now rescue nil
+      when Numeric
+        # NOTE: the concept of awaiting gracefully might/should also be used with the pool/client closing
+        millis = java.util.concurrent.TimeUnit::MILLISECONDS
+        @executor&.await_termination(await * 1000, millis) rescue nil
+      else
+        nil
+      end
+      @async_requests.close
     end
 
+    # @private
+    class Finalizer
+
+      def initialize
+        @_objs = []
+      end
+
+      def push(obj, args)
+        @_objs.unshift [java.lang.ref.WeakReference.new(obj), Array(args)]
+      end
+
+      def call(id = nil) # when called on finalization an object id arg is passed
+        @_objs.each { |obj, args| obj.get&.send(*args) rescue nil }
+      end
+
+    end
+    private_constant :Finalizer
+
     # Get at the underlying ExecutorService used to invoke asynchronous calls.
     def executor
-      create_executor_if_needed
-      @executor
+      @executor ||= create_executor
     end
 
-    def self.shutdown_on_finalize(client, objs)
-      ObjectSpace.define_finalizer client, -> {
-                                     objs.each { |obj, args| obj.send(*args) rescue nil }
-                                   }
+    # @private
+    def self.shutdown_on_finalize(client, finalizer)
+      ObjectSpace.define_finalizer(client, finalizer)
     end
 
     protected
@@ -366,8 +423,9 @@ module Manticore
     # Takes an object and a message to pass to the object to destroy it. This is done rather than
     # a proc to avoid creating a closure that would maintain a reference to this client, which
     # would prevent the client from being cleaned up.
+    # @private
     def finalize(object, args)
-      @finalizers << [WeakRef.new(object), Array(args)]
+      @finalizer.push(object, args)
     end
 
     def url_as_regex(url)
@@ -387,7 +445,7 @@ module Manticore
 
       # :nocov:
       if options[:ignore_ssl_validation]
-        $stderr.puts "The options[:ignore_ssl_validation] setting is deprecated in favor of options[:ssl][:verify]"
+        warn "The options[:ignore_ssl_validation] setting is deprecated in favor of options[:ssl][:verify]"
         options[:ssl] ||= {}
         options[:ssl] = {:verify => !options.delete(:ignore_ssl_validation)}.merge(options[:ssl])
       end
@@ -419,17 +477,16 @@ module Manticore
       socket_config_builder.build
     end
 
-    def create_executor_if_needed
-      return @executor if @executor
-      @executor = Executors.new_cached_thread_pool(ExecutorThreadFactory.new)
-      finalize @executor, :shutdown
+    def create_executor
+      executor = Executors.new_cached_thread_pool(ExecutorThreadFactory.new(self))
+      finalize executor, :shutdown
+      executor
     end
 
     def request(klass, url, options, &block)
       req, context = request_from_options(klass, url, options)
       async = options.delete(:async)
       background = options.delete(:async_background)
-      create_executor_if_needed if (background || async)
       response = response_object_for(req, context, &block)
 
       if async
@@ -524,7 +581,6 @@ module Manticore
     end
 
     def get_proxy_host(opt)
-      host = nil
       if opt.is_a? String
         uri = URI.parse(opt)
         if uri.host
@@ -609,20 +665,27 @@ module Manticore
 
     # Configure the SSL Context
     def ssl_socket_factory_from_options(ssl_options)
-      trust_store = trust_strategy = nil
-
-      verifier = SSLConnectionSocketFactory::STRICT_HOSTNAME_VERIFIER
-      case ssl_options.fetch(:verify, :strict)
-      when false, :disable, :none
-        trust_store = nil
-        trust_strategy = TrustSelfSignedStrategy.new
+      trust_strategy = nil
+
+      case ssl_options.fetch(:verify, :default)
+      when :none, :disable
+        trust_strategy = TrustAllStrategy::INSTANCE
+        verifier = NoopHostnameVerifier::INSTANCE
+      when false # compatibility
+        trust_strategy = TrustSelfSignedStrategy::INSTANCE
         verifier = SSLConnectionSocketFactory::ALLOW_ALL_HOSTNAME_VERIFIER
       when :browser
         verifier = SSLConnectionSocketFactory::BROWSER_COMPATIBLE_HOSTNAME_VERIFIER
-      when true, :strict, :default
+      when :default, true
+        verifier = DefaultHostnameVerifier.new
+      when :strict # compatibility
         verifier = SSLConnectionSocketFactory::STRICT_HOSTNAME_VERIFIER
       else
-        raise "Invalid value for :verify. Valid values are (:all, :browser, :default)"
+        raise "Invalid value for :verify. Valid values are (:default, :browser, :none)"
+      end
+
+      if ssl_options.include?(:trust_strategy)
+        trust_strategy = TrustStrategies.combine(trust_strategy, ssl_options.fetch(:trust_strategy))
       end
 
       context = SSLContextBuilder.new
@@ -646,7 +709,7 @@ module Manticore
         end
       end
 
-      context.load_trust_material(trust_store, trust_strategy)
+      context.java_send :loadTrustMaterial, [KeyStore, TrustStrategy], trust_store, trust_strategy
     end
 
     KEY_EXTRACTION_REGEXP = /(?:^-----BEGIN(.* )PRIVATE KEY-----\n)(.*?)(?:-----END\1PRIVATE KEY.*$)/m
@@ -658,7 +721,6 @@ module Manticore
       # Support OpenSSL-style bare X.509 certs with an RSA key
       if ssl_options[:client_cert] && ssl_options[:client_key]
         key_store ||= blank_keystore
-        certs, key = nil, nil
 
         cert_str = if ssl_options[:client_cert].is_a?(OpenSSL::X509::Certificate)
                      ssl_options[:client_cert].to_s
@@ -697,7 +759,7 @@ module Manticore
     def get_store(prefix, options)
       KeyStore.get_instance(options[:"#{prefix}_type"] || guess_store_type(options[prefix])).tap do |store|
         instream = open(options[prefix], "rb").to_inputstream
-        store.load(instream, options.fetch(:"#{prefix}_password", nil).to_java.toCharArray)
+        store.load(instream, options.fetch(:"#{prefix}_password", nil).to_java&.toCharArray)
       end
     end
 
@@ -722,10 +784,11 @@ module Manticore
     end
   end
 
-  class LoggingStandardRetryHandler < Java::OrgApacheHttpImplClient::StandardHttpRequestRetryHandler
+  class LoggingStandardRetryHandler < org.apache.http.impl.client.StandardHttpRequestRetryHandler
     def retryRequest(exception, executionCount, context)
       context.setAttribute "retryCount", executionCount
       super(exception, executionCount, context)
     end
   end
+  private_constant :LoggingStandardRetryHandler
 end

data/lib/manticore/version.rb

@@ -1,3 +1,3 @@
 module Manticore
-  VERSION = "0.7.1"
+  VERSION = "0.9.1"
 end

data/lib/manticore.rb

@@ -67,6 +67,7 @@ module Manticore
 
   require_relative "./manticore/java_extensions"
   require_relative "./manticore/client/proxies"
+  require_relative "./manticore/client/trust_strategies"
   require_relative "./manticore/client"
   require_relative "./manticore/response"
   require_relative "./manticore/stubbed_response"

data/lib/manticore_jars.rb

@@ -1,8 +1,18 @@
 # this is a generated file, to avoid over-writing it just delete this comment
-require "jar_dependencies"
+begin
+  require 'jar_dependencies'
+rescue LoadError
+  require 'commons-logging/commons-logging/1.2/commons-logging-1.2.jar'
+  require 'commons-codec/commons-codec/1.15/commons-codec-1.15.jar'
+  require 'org/apache/httpcomponents/httpcore/4.4.14/httpcore-4.4.14.jar'
+  require 'org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar'
+  require 'org/apache/httpcomponents/httpmime/4.5.13/httpmime-4.5.13.jar'
+end
 
-require_jar("commons-logging", "commons-logging", "1.2")
-require_jar("org.apache.httpcomponents", "httpmime", "4.5.2")
-require_jar("commons-codec", "commons-codec", "1.10")
-require_jar("org.apache.httpcomponents", "httpclient", "4.5.2")
-require_jar("org.apache.httpcomponents", "httpcore", "4.4.4")
+if defined? Jars
+  require_jar 'commons-logging', 'commons-logging', '1.2'
+  require_jar 'commons-codec', 'commons-codec', '1.15'
+  require_jar 'org.apache.httpcomponents', 'httpcore', '4.4.14'
+  require_jar 'org.apache.httpcomponents', 'httpclient', '4.5.13'
+  require_jar 'org.apache.httpcomponents', 'httpmime', '4.5.13'
+end

data/manticore.gemspec

@@ -29,13 +29,11 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "openssl_pkcs8_pure"
 
-  spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake"
   spec.add_development_dependency "jar-dependencies", "~> 0.4.1"
 
-  spec.requirements << "jar org.apache.httpcomponents:httpclient, '~> 4.5.0'"
-  spec.requirements << "jar org.apache.httpcomponents:httpmime,   '~> 4.5.0'"
+  spec.requirements << "jar org.apache.httpcomponents:httpclient, '~> 4.5.13'"
+  spec.requirements << "jar org.apache.httpcomponents:httpmime,   '~> 4.5.13'"
   spec.requirements << "jar commons-logging:commons-logging,      '~> 1.2'"
   spec.requirements << "jar commons-codec:commons-codec,          '~> 1.9'"
-  spec.requirements << "jar org.apache.httpcomponents:httpcore,   '~> 4.4.4'"
+  spec.requirements << "jar org.apache.httpcomponents:httpcore,   '~> 4.4.14'"
 end

data/spec/manticore/client_spec.rb

@@ -1,12 +1,11 @@
 # encoding: utf-8
 require "spec_helper"
 
-java_import "org.apache.http.entity.mime.MultipartEntityBuilder"
-java_import "org.apache.http.entity.ContentType"
-
 describe Manticore::Client do
   let(:client) { Manticore::Client.new }
 
+  after { client.close }
+
   it "fetches a URL and return a response" do
     expect(client.get(local_server)).to be_a Manticore::Response
   end
@@ -96,7 +95,7 @@ describe Manticore::Client do
 
   describe "ignore_ssl_validation (deprecated option)" do
     context "when on" do
-      let(:client) { Manticore::Client.new ssl: {verify: false} }
+      let(:client) { Manticore::Client.new ssl: { verify: false } }
 
       it "does not break on SSL validation errors" do
         expect { client.get("https://localhost:55444/").body }.to_not raise_exception
@@ -104,7 +103,7 @@ describe Manticore::Client do
     end
 
     context "when off" do
-      let(:client) { Manticore::Client.new ssl: {verify: true} }
+      let(:client) { Manticore::Client.new ssl: { verify: true } }
 
       it "breaks on SSL validation errors" do
         expect { client.get("https://localhost:55444/").call }.to raise_exception(Manticore::ClientProtocolException)
@@ -148,6 +147,29 @@ describe Manticore::Client do
         end
       end
 
+      context "when on and custom trust strategy is given" do
+        # let(:custom_trust_strategy) { Proc.new {|chain,type| true } }
+        let(:client) { Manticore::Client.new :ssl => {:verify => :strict, :trust_strategy => custom_trust_strategy} }
+        context 'and trust strategy approves the cert chain' do
+          let(:custom_trust_strategy) { Proc.new { |chain,type| true } }
+          it "verifies the request and succeed" do
+            expect { client.get("https://localhost:55444/").body }.to_not raise_exception
+          end
+        end
+        context 'and trust strategy does not approve the cert chain' do
+          let(:custom_trust_strategy) { Proc.new { |chain,type| false } }
+          it "breaks on SSL validation errors" do
+            begin
+              client.get("https://localhost:55445/").body
+            rescue Manticore::ClientProtocolException => e
+              expect( e.cause ).to be_a javax.net.ssl.SSLHandshakeException
+            else
+              fail "exception not raised"
+            end
+          end
+        end
+      end
+
       context "when the client specifies a protocol list" do
         let(:client) { Manticore::Client.new :ssl => {verify: :strict, truststore: File.expand_path("../../ssl/truststore.jks", __FILE__), truststore_password: "test123", protocols: ["TLSv1", "TLSv1.1", "TLSv1.2"]} }
 
@@ -157,10 +179,10 @@ describe Manticore::Client do
       end
 
       context "when on and custom trust store is given with the wrong password" do
-        let(:client) { Manticore::Client.new :ssl => {verify: :strict, truststore: File.expand_path("../../ssl/truststore.jks", __FILE__), truststore_password: "wrongpass"} }
+        let(:ssl_opts) { { verify: :strict, truststore: File.expand_path("../../ssl/truststore.jks", __FILE__), truststore_password: "wrongpass" } }
 
         it "fails to load the keystore" do
-          expect { client.get("https://localhost:55444/").body }.to raise_exception(Java::JavaIo::IOException)
+          expect { Manticore::Client.new(:ssl => ssl_opts) }.to raise_exception(Java::JavaIo::IOException)
         end
       end
 
@@ -193,7 +215,7 @@ describe Manticore::Client do
         let(:client) {
           Manticore::Client.new(
             :ssl => {
-              verify: :strict,
+              verify: :default,
               ca_file: File.expand_path("../../ssl/root-ca.crt", __FILE__),
               client_cert: OpenSSL::X509::Certificate.new(File.read(File.expand_path("../../ssl/client.crt", __FILE__))),
               client_key: OpenSSL::PKey::RSA.new(File.read(File.expand_path("../../ssl/client.key", __FILE__))),
@@ -210,7 +232,7 @@ describe Manticore::Client do
         let(:client) {
           Manticore::Client.new(
             :ssl => {
-              verify: :strict,
+              verify: :default,
               ca_file: File.expand_path("../../ssl/root-ca.crt", __FILE__),
               client_cert: File.read(File.expand_path("../../ssl/client.crt", __FILE__)),
               client_key: File.read(File.expand_path("../../ssl/client.key", __FILE__)),
@@ -233,6 +255,27 @@ describe Manticore::Client do
         it "does not break on expired SSL certificates" do
           expect { client.get("https://localhost:55446/").body }.to_not raise_exception
         end
+
+        it "does not break on untrusted certificates" do
+          expect { client.get("https://localhost:55447/").body }.to_not raise_exception
+        end
+
+        context "when custom trust strategy is given" do
+          # let(:custom_trust_strategy) { Proc.new {|chain,type| true } }
+          let(:client) { Manticore::Client.new :ssl => {:verify => :disable, :trust_strategy => custom_trust_strategy} }
+          context 'and trust strategy approves the cert chain' do
+            let(:custom_trust_strategy) { Proc.new { |chain,type| true } }
+            it "verifies the request and succeed" do
+              expect { client.get("https://localhost:55444/").body }.to_not raise_exception
+            end
+          end
+          context 'and trust strategy does not approve the cert chain' do
+            let(:custom_trust_strategy) { Proc.new { |chain,type| false } }
+            it "verifies the request and succeed" do
+              expect { client.get("https://localhost:55444/").body }.to_not raise_exception
+            end
+          end
+        end
       end
 
       context "against a server that verifies clients" do
@@ -272,11 +315,11 @@ describe Manticore::Client do
     end
 
     describe ":cipher_suites" do
-      skip
+      skip 'TODO: someone should write the spec'
     end
 
     describe ":protocols" do
-      skip
+      skip 'TODO: someone should write the spec'
     end
   end
 
@@ -531,7 +574,9 @@ describe Manticore::Client do
 
     it "sends an arbitrary entity" do
       f = open(File.expand_path(File.join(__FILE__, "..", "..", "spec_helper.rb")), "r").to_inputstream
-      multipart_entity = MultipartEntityBuilder.create.add_text_body("foo", "bar").add_binary_body("whatever", f, ContentType::TEXT_PLAIN, __FILE__)
+      multipart_entity = org.apache.http.entity.mime.MultipartEntityBuilder.create.
+          add_text_body("foo", "bar").
+          add_binary_body("whatever", f, org.apache.http.entity.ContentType::TEXT_PLAIN, __FILE__)
       response = client.post(local_server, entity: multipart_entity.build)
       expect(response.body).to match "RSpec.configure"
     end
@@ -742,14 +787,13 @@ describe Manticore::Client do
   context "with a misbehaving endpoint" do
     let(:port) do
       p = 4000
-      server = nil
       begin
         server = TCPServer.new p
       rescue Errno::EADDRINUSE
         p += 1
         retry
       ensure
-        server.close
+        server&.close
       end
       p
     end
@@ -770,6 +814,7 @@ describe Manticore::Client do
             ].join("\n"))
             client.close
           rescue IOError => e
+            warn "caught an error: #{e.inspect}"
             break
           end
         end

data/spec/manticore/client_trust_strategies_spec.rb

@@ -0,0 +1,168 @@
+# encoding: utf-8
+require "spec_helper"
+describe Manticore::Client::TrustStrategies do
+  describe '#coerce' do
+    subject(:coerced) { described_class.coerce(input) }
+    context 'with a nil value' do
+      let(:input) { nil }
+      it 'returns the value unchanged' do
+        expect(coerced).to be_nil
+      end
+    end
+    context 'with an implementation of org.apache.http.conn.ssl.TrustStrategy' do
+      let(:input) { org.apache.http.conn.ssl.TrustAllStrategy::INSTANCE }
+      it 'returns the value unchanged' do
+        expect(coerced).to be input
+      end
+    end
+    context 'with a Proc' do
+      let(:input) { ->(chain, type) { true } }
+      it 'wraps the proc in a `CustomTrustStrategy`' do
+        expect(Manticore::Client::CustomTrustStrategy).to receive(:new).with(input).and_call_original
+        expect(described_class.coerce(input)).to be_a_kind_of Manticore::Client::CustomTrustStrategy
+      end
+    end
+  end
+
+  describe '#combine' do
+    context 'when left-hand value is nil' do
+      let(:left_hand_strategy) { nil }
+      let(:right_hand_strategy) { described_class.coerce(->(chain,type){ true }) }
+      it 'returns the right-hand value coerced' do
+        expect(described_class).to receive(:coerce).with(right_hand_strategy).and_call_original
+        expect(described_class.combine(left_hand_strategy, right_hand_strategy)).to be right_hand_strategy
+      end
+    end
+    context 'when the right-hand value is nil' do
+      let(:left_hand_strategy) {  described_class.coerce(->(chain,type){ true }) }
+      let(:right_hand_strategy) { nil }
+      it 'returns the left-hand value coerced' do
+        expect(described_class).to receive(:coerce).with(left_hand_strategy).and_call_original
+        expect(described_class.combine(left_hand_strategy, right_hand_strategy)).to be left_hand_strategy
+      end
+    end
+    context 'when neither value is nil' do
+      let(:left_hand_strategy) {  described_class.coerce(->(chain,type){ true }) }
+      let(:right_hand_strategy) { described_class.coerce(->(chain,type){ true }) }
+
+      it 'returns a CombinedTrustStrategy' do
+        expect(Manticore::Client::CombinedTrustStrategy)
+          .to receive(:new).with(left_hand_strategy, right_hand_strategy).and_call_original
+
+        # ensures that the values are coerced.
+        expect(described_class).to receive(:coerce).with(left_hand_strategy).and_call_original
+        expect(described_class).to receive(:coerce).with(right_hand_strategy).and_call_original
+
+        combined = described_class.combine(left_hand_strategy, right_hand_strategy)
+        expect(combined).to be_a_kind_of Manticore::Client::CombinedTrustStrategy
+      end
+    end
+    context 'when both values are nil' do
+      let(:left_hand_strategy) { nil }
+      let(:right_hand_strategy) { nil }
+
+      it 'returns nil' do
+        expect(described_class.combine(left_hand_strategy, right_hand_strategy)).to be nil
+      end
+    end
+  end
+end
+
+describe Manticore::Client::CustomTrustStrategy do
+
+  subject(:custom_trust_strategy) { described_class.new(trust_strategy_proc) }
+
+  context 'when called via Java interface' do
+    def load_java_cert(file_path)
+      pem_contents = File.read(file_path)
+      cf = java.security.cert.CertificateFactory::getInstance("X.509")
+      is = java.io.ByteArrayInputStream.new(pem_contents.to_java_bytes)
+      cf.generateCertificate(is)
+    end
+
+    let(:java_host_cert) { load_java_cert(File.expand_path("../../ssl/host.crt", __FILE__)) }
+    let(:java_root_cert) { load_java_cert(File.expand_path("../../ssl/root-ca.crt", __FILE__)) }
+    let(:java_chain) { [java_host_cert, java_root_cert].to_java(java.security.cert.X509Certificate) }
+    let(:java_type) { java.lang.String.new("my_type".to_java_bytes) }
+
+    subject(:java_trust_strategy) { custom_trust_strategy.to_java(org.apache.http.conn.ssl.TrustStrategy) }
+
+    context 'when called with Java Certs and a Java String' do
+      let(:trust_strategy_proc) { ->(chain,type) { true } }
+      it 'yields an enum of equivalent Ruby certs and an equivalent Ruby String' do
+        expect(trust_strategy_proc).to receive(:call) do |chain, type|
+          expect(chain.to_a.length).to eq java_chain.length
+          chain.each_with_index do |cert, idx|
+            expect(cert).to be_a_kind_of OpenSSL::X509::Certificate
+            expect(cert.to_der).to eq String.from_java_bytes(java_chain[idx].encoded)
+          end
+          expect(type).to be_a_kind_of String
+          expect(type).to eq String.from_java_bytes(java_type.bytes)
+        end
+
+        expect(java_trust_strategy.isTrusted(java_chain, java_type)).to be true
+      end
+    end
+
+    context 'when the ruby block returns false' do
+      let(:trust_strategy_proc) { ->(chain,type) { false } }
+      it 'returns false' do
+        expect(java_trust_strategy.isTrusted(java_chain, java_type)).to be false
+      end
+    end
+
+    context 'when the ruby block returns true' do
+      let(:trust_strategy_proc) { ->(chain,type) { true } }
+      it 'returns true' do
+        expect(java_trust_strategy.isTrusted(java_chain, java_type)).to be true
+      end
+    end
+
+    context 'when the ruby block raises an exception' do
+      let(:trust_strategy_proc) { ->(chain, type) { fail(OpenSSL::X509::CertificateError, 'intentional') } }
+      it 'throws a CertificateException' do
+        expect {
+          java_trust_strategy.isTrusted(java_chain, java_type)
+        }.to raise_exception(java.security.cert.CertificateException)
+      end
+    end
+  end
+end
+
+describe Manticore::Client::CombinedTrustStrategy do
+  let(:always_trust_strategy) { ->(chain,type) { true } }
+  let(:never_trust_strategy) { ->(chain,type) { false } }
+
+  subject(:combined_trust_strategy) { Manticore::Client::TrustStrategies.combine(left_hand_strategy, right_hand_strategy) }
+
+  context 'when left-hand strategy trusts' do
+    let(:left_hand_strategy) { always_trust_strategy }
+    context 'when right-hand strategy trusts' do
+      let(:right_hand_strategy) { always_trust_strategy }
+      it 'trusts' do
+        expect(combined_trust_strategy.trusted?([],'ignored')).to be true
+      end
+    end
+    context 'when right-hand strategy does not trust' do
+      let(:right_hand_strategy) { never_trust_strategy }
+      it 'trusts' do
+        expect(combined_trust_strategy.trusted?([],'ignored')).to be true
+      end
+    end
+  end
+  context 'when left-hand strategy does not trust' do
+    let(:left_hand_strategy) { never_trust_strategy }
+    context 'when right-hand strategy trusts' do
+      let(:right_hand_strategy) { always_trust_strategy }
+      it 'trusts' do
+        expect(combined_trust_strategy.trusted?([],'ignored')).to be true
+      end
+    end
+    context 'when right-hand strategy does not trust' do
+      let(:right_hand_strategy) { never_trust_strategy }
+      it 'does not trust' do
+        expect(combined_trust_strategy.trusted?([],'ignored')).to be false
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -151,6 +151,7 @@ RSpec.configure do |c|
     start_ssl_server 55444
     start_ssl_server 55445, :SSLVerifyClient => OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT, :SSLCACertificateFile => File.expand_path("../ssl/root-ca.crt", __FILE__)
     start_ssl_server 55446, cert: File.expand_path("../ssl/host-expired.crt", __FILE__)
+    start_ssl_server 55447, cert: File.expand_path("../ssl/host-untrusted.crt", __FILE__), SSLCACertificateFile: File.expand_path("../ssl/root-untrusted-ca.crt", __FILE__)
 
     Manticore.disable_httpcomponents_logging!
   }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: manticore
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.9.1
 platform: java
 authors:
 - Chris Heald
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-18 00:00:00.000000000 Z
+date: 2022-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -17,36 +17,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: openssl_pkcs8_pure
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: bundler
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: rake
-  type: :development
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -59,8 +31,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.4.1
   name: jar-dependencies
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -86,12 +58,13 @@ files:
 - ext/manticore/org/manticore/HttpGetWithEntity.java
 - ext/manticore/org/manticore/Manticore.java
 - gem-public_cert.pem
-- lib/commons-codec/commons-codec/1.10/commons-codec-1.10.jar
+- lib/commons-codec/commons-codec/1.15/commons-codec-1.15.jar
 - lib/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
 - lib/faraday/adapter/manticore.rb
 - lib/manticore.rb
 - lib/manticore/client.rb
 - lib/manticore/client/proxies.rb
+- lib/manticore/client/trust_strategies.rb
 - lib/manticore/cookie.rb
 - lib/manticore/facade.rb
 - lib/manticore/java_extensions.rb
@@ -99,13 +72,14 @@ files:
 - lib/manticore/stubbed_response.rb
 - lib/manticore/version.rb
 - lib/manticore_jars.rb
-- lib/org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar
-- lib/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar
-- lib/org/apache/httpcomponents/httpmime/4.5.2/httpmime-4.5.2.jar
+- lib/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar
+- lib/org/apache/httpcomponents/httpcore/4.4.14/httpcore-4.4.14.jar
+- lib/org/apache/httpcomponents/httpmime/4.5.13/httpmime-4.5.13.jar
 - lib/org/manticore/manticore-ext.jar
 - manticore.gemspec
 - spec/manticore/client_proxy_spec.rb
 - spec/manticore/client_spec.rb
+- spec/manticore/client_trust_strategies_spec.rb
 - spec/manticore/cookie_spec.rb
 - spec/manticore/facade_spec.rb
 - spec/manticore/response_spec.rb
@@ -131,18 +105,19 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- jar org.apache.httpcomponents:httpclient, '~> 4.5.0'
-- jar org.apache.httpcomponents:httpmime,   '~> 4.5.0'
+- jar org.apache.httpcomponents:httpclient, '~> 4.5.13'
+- jar org.apache.httpcomponents:httpmime,   '~> 4.5.13'
 - jar commons-logging:commons-logging,      '~> 1.2'
 - jar commons-codec:commons-codec,          '~> 1.9'
-- jar org.apache.httpcomponents:httpcore,   '~> 4.4.4'
-rubygems_version: 3.0.6
+- jar org.apache.httpcomponents:httpcore,   '~> 4.4.14'
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Manticore is an HTTP client built on the Apache HttpCore components
 test_files:
 - spec/manticore/client_proxy_spec.rb
 - spec/manticore/client_spec.rb
+- spec/manticore/client_trust_strategies_spec.rb
 - spec/manticore/cookie_spec.rb
 - spec/manticore/facade_spec.rb
 - spec/manticore/response_spec.rb