mongo 2.13.0.beta1 → 2.14.0

data/lib/mongo/server_selector/nearest.rb

@@ -20,7 +20,6 @@ module Mongo
     #
     # @since 2.0.0
     class Nearest < Base
-      include Selectable
 
       # Name of the this read preference in the server's format.
       #
@@ -92,14 +91,10 @@ module Mongo
       # Select the near servers taking into account any defined tag sets and
       #   local threshold between the nearest server and other servers.
       #
-      # @example Select nearest servers given a list of candidates.
-      #   preference = Mongo::ServerSelector::Nearest.new
-      #   preference.select_server(cluster)
-      #
       # @return [ Array ] The nearest servers from the list of candidates.
       #
       # @since 2.0.0
-      def select(candidates)
+      def select_in_replica_set(candidates)
         matching_servers = filter_stale_servers(candidates, primary(candidates).first)
         matching_servers = match_tag_sets(matching_servers) unless tag_sets.empty?
         near_servers(matching_servers)

data/lib/mongo/server_selector/primary.rb

@@ -21,7 +21,6 @@ module Mongo
     #
     # @since 2.0.0
     class Primary < Base
-      include Selectable
 
       # Name of the this read preference in the server's format.
       #
@@ -94,14 +93,10 @@ module Mongo
 
       # Select the primary server from a list of candidates.
       #
-      # @example Select the primary server given a list of candidates.
-      #   preference = Mongo::ServerSelector::Primary.new
-      #   preference.select([candidate_1, candidate_2])
-      #
       # @return [ Array ] The primary server from the list of candidates.
       #
       # @since 2.0.0
-      def select(candidates)
+      def select_in_replica_set(candidates)
         primary(candidates)
       end
 

data/lib/mongo/server_selector/primary_preferred.rb

@@ -21,7 +21,6 @@ module Mongo
     #
     # @since 2.0.0
     class PrimaryPreferred < Base
-      include Selectable
 
       # Name of the this read preference in the server's format.
       #
@@ -93,19 +92,17 @@ module Mongo
       # Select servers taking into account any defined tag sets and
       #   local threshold, with the primary preferred.
       #
-      # @example Select servers given a list of candidates,
-      #   with the primary preferred.
-      #   preference = Mongo::ServerSelector::PrimaryPreferred.new
-      #   preference.select([candidate_1, candidate_2])
-      #
       # @return [ Array ] A list of servers matching tag sets and acceptable
       #   latency with the primary preferred.
       #
       # @since 2.0.0
-      def select(candidates)
-        primary = primary(candidates)
-        secondaries = near_servers(secondaries(candidates))
-        primary.first ? primary : secondaries
+      def select_in_replica_set(candidates)
+        primaries = primary(candidates)
+        if primaries.first
+          primaries
+        else
+          near_servers(secondaries(candidates))
+        end
       end
 
       def max_staleness_allowed?

data/lib/mongo/server_selector/secondary.rb

@@ -21,7 +21,6 @@ module Mongo
     #
     # @since 2.0.0
     class Secondary < Base
-      include Selectable
 
       # Name of the this read preference in the server's format.
       #
@@ -93,14 +92,10 @@ module Mongo
       # Select the secondary servers taking into account any defined tag sets and
       #   local threshold between the nearest secondary and other secondaries.
       #
-      # @example Select secondary servers given a list of candidates.
-      #   preference = Mongo::ServerSelector::Secondary.new
-      #   preference.select([candidate_1, candidate_2])
-      #
       # @return [ Array ] The secondary servers from the list of candidates.
       #
       # @since 2.0.0
-      def select(candidates)
+      def select_in_replica_set(candidates)
         near_servers(secondaries(candidates))
       end
 

data/lib/mongo/server_selector/secondary_preferred.rb

@@ -21,7 +21,6 @@ module Mongo
     #
     # @since 2.0.0
     class SecondaryPreferred < Base
-      include Selectable
 
       # Name of the this read preference in the server's format.
       #
@@ -101,16 +100,11 @@ module Mongo
       # Select servers taking into account any defined tag sets and
       #   local threshold, with secondaries.
       #
-      # @example Select servers given a list of candidates,
-      #   with secondaries preferred.
-      #   preference = Mongo::ServerSelector::SecondaryPreferred.new
-      #   preference.select([candidate_1, candidate_2])
-      #
       # @return [ Array ] A list of servers matching tag sets and acceptable
       #   latency with secondaries preferred.
       #
       # @since 2.0.0
-      def select(candidates)
+      def select_in_replica_set(candidates)
         near_servers(secondaries(candidates)) + primary(candidates)
       end
 

data/lib/mongo/session.rb

@@ -227,7 +227,9 @@ module Mongo
     # Error message describing that sessions are not supported by the server version.
     #
     # @since 2.5.0
+    # @deprecated
     SESSIONS_NOT_SUPPORTED = 'Sessions are not supported by the connected servers.'.freeze
+    # Note: SESSIONS_NOT_SUPPORTED is used by Mongoid - do not remove from driver.
 
     # The state of a session in which the last operation was not related to
     # any transaction or no operations have yet occurred.
@@ -377,6 +379,7 @@ module Mongo
           rv = yield self
         rescue Exception => e
           if within_states?(STARTING_TRANSACTION_STATE, TRANSACTION_IN_PROGRESS_STATE)
+            log_warn("Aborting transaction due to #{e.class}: #{e}")
             abort_transaction
             transaction_in_progress = false
           end
@@ -441,7 +444,7 @@ module Mongo
       true
     ensure
       if transaction_in_progress
-        log_warn('with_transaction callback altered with_transaction loop, aborting transaction')
+        log_warn('with_transaction callback broke out of with_transaction loop, aborting transaction')
         begin
           abort_transaction
         rescue Error::OperationFailure, Error::InvalidTransactionOperation
@@ -532,6 +535,7 @@ module Mongo
     #
     # @since 2.6.0
     def commit_transaction(options=nil)
+      QueryCache.clear
       check_if_ended!
       check_if_no_transaction!
 
@@ -600,6 +604,8 @@ module Mongo
     #
     # @since 2.6.0
     def abort_transaction
+      QueryCache.clear
+
       check_if_ended!
       check_if_no_transaction!
 

data/lib/mongo/socket.rb

@@ -15,19 +15,22 @@
 require 'mongo/socket/ssl'
 require 'mongo/socket/tcp'
 require 'mongo/socket/unix'
+require 'mongo/socket/ocsp_verifier'
+require 'mongo/socket/ocsp_cache'
 
 module Mongo
 
   # Provides additional data around sockets for the driver's use.
   #
   # @since 2.0.0
+  # @api private
   class Socket
     include ::Socket::Constants
 
-    # Error message for SSL related exceptions.
+    # Error message for TLS related exceptions.
     #
     # @since 2.0.0
-    SSL_ERROR = 'MongoDB may not be configured with SSL support'.freeze
+    SSL_ERROR = 'MongoDB may not be configured with TLS support'.freeze
 
     # Error message for timeouts on socket calls.
     #
@@ -105,9 +108,14 @@ module Mongo
     def summary
       fileno = @socket&.fileno rescue '<no socket>' || '<no socket>'
       if monitor?
-        "#{connection_address}:m #{fileno}"
+        indicator = if options[:push]
+          'pm'
+        else
+          'm'
+        end
+        "#{connection_address};#{indicator};fd=#{fileno}"
       else
-        "#{connection_address}:c:#{connection_generation} #{fileno}"
+        "#{connection_address};c:#{connection_generation};fd=#{fileno}"
       end
     end
 
@@ -123,7 +131,7 @@ module Mongo
       sock_arr = [ @socket ]
       if Kernel::select(sock_arr, nil, sock_arr, 0)
         # The eof? call is supposed to return immediately since select
-        # indicated the socket is readable. However, if @socket is an SSL
+        # indicated the socket is readable. However, if @socket is a TLS
         # socket, eof? can block anyway - see RUBY-2140.
         begin
           Timeout.timeout(0.1) do
@@ -173,20 +181,21 @@ module Mongo
     #   socket.read(4096)
     #
     # @param [ Integer ] length The number of bytes to read.
+    # @param [ Numeric ] timeout The timeout to use for each chunk read.
     #
     # @raise [ Mongo::SocketError ] If not all data is returned.
     #
     # @return [ Object ] The data from the socket.
     #
     # @since 2.0.0
-    def read(length)
+    def read(length, timeout: nil)
       map_exceptions do
-        data = read_from_socket(length)
+        data = read_from_socket(length, timeout: timeout)
         unless (data.length > 0 || length == 0)
           raise IOError, "Expected to read > 0 bytes but read 0 bytes"
         end
         while data.length < length
-          chunk = read_from_socket(length - data.length)
+          chunk = read_from_socket(length - data.length, timeout: timeout)
           unless (chunk.length > 0 || length == 0)
             raise IOError, "Expected to read > 0 bytes but read 0 bytes"
           end
@@ -243,14 +252,19 @@ module Mongo
 
     private
 
-    def read_from_socket(length)
+    def read_from_socket(length, timeout: nil)
       # Just in case
       if length == 0
         return ''.force_encoding('BINARY')
       end
 
-      if _timeout = self.timeout
-        deadline = Time.now + _timeout
+      _timeout = timeout || self.timeout
+      if _timeout
+        if _timeout > 0
+          deadline = Time.now + _timeout
+        elsif _timeout < 0
+          raise Errno::ETIMEDOUT, "Negative timeout #{_timeout} given to socket"
+        end
       end
 
       # We want to have a fixed and reasonably small size buffer for reads
@@ -330,7 +344,7 @@ module Mongo
     end
 
     def read_buffer_size
-      # Buffer size for non-SSL reads
+      # Buffer size for non-TLS reads
       # 64kb
       65536
     end

data/lib/mongo/socket/ocsp_cache.rb

@@ -0,0 +1,97 @@
+# Copyright (C) 2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Socket
+
+    # This module caches OCSP responses for their indicated validity time.
+    #
+    # The key is the CertificateId used for the OCSP request.
+    # The value is the SingleResponse on Ruby 2.4+, or the OpenStruct
+    # emulation of it on Ruby 2.3.
+    #
+    # @api private
+    module OcspCache
+      module_function def set(cert_id, response)
+        delete(cert_id)
+        responses << response
+      end
+
+      # Retrieves a cached SingleResponse for the specified CertificateId.
+      #
+      # This method may return expired responses if they are revoked.
+      # Such responses were valid when they were first received.
+      #
+      # This method may also return responses that are valid but that may
+      # expire by the time caller uses them. The caller should not perform
+      # update time checks on the returned response.
+      #
+      # @return [ OpenSSL::OCSP::SingleResponse | OpenStruct ] The previously
+      #   retrieved response.
+      module_function def get(cert_id)
+        resp = responses.detect do |resp|
+          resp.certid.cmp(cert_id)
+        end
+        if resp
+          # Only expire responses with good status.
+          # Once a certificate is revoked, it should stay revoked forever,
+          # hence we should be able to cache revoked responses indefinitely.
+          if resp.cert_status == OpenSSL::OCSP::V_CERTSTATUS_GOOD &&
+            resp.next_update < Time.now
+          then
+            responses.delete(resp)
+            resp = nil
+          end
+        end
+
+        # If we have connected to a server and cached the OCSP response for it,
+        # and then never connect to that server again, the cached OCSP response
+        # is going to remain in memory indefinitely. Periodically remove all
+        # expired OCSP responses, not just the ones matching the certificate id
+        # we are querying by.
+        if rand < 0.01
+          responses.delete_if do |resp|
+            resp.next_update < Time.now
+          end
+        end
+
+        resp
+      end
+
+      module_function def delete(cert_id)
+        responses.delete_if do |resp|
+          resp.certid.cmp(cert_id)
+        end
+      end
+
+      # Clears the driver's OCSP response cache.
+      #
+      # @note Use Mongo.clear_ocsp_cache from applications instead of invoking
+      #   this method directly.
+      module_function def clear
+        responses.replace([])
+      end
+
+      private
+
+      LOCK = Mutex.new
+
+      module_function def responses
+        LOCK.synchronize do
+          @responses ||= []
+        end
+      end
+    end
+  end
+end

data/lib/mongo/socket/ocsp_verifier.rb

@@ -0,0 +1,368 @@
+# Copyright (C) 2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Net
+  autoload :HTTP, 'net/http'
+end
+
+module Mongo
+  class Socket
+
+    # OCSP endpoint verifier.
+    #
+    # After a TLS connection is established, this verifier inspects the
+    # certificate presented by the server, and if the certificate contains
+    # an OCSP URI, performs the OCSP status request to the specified URI
+    # (following up to 5 redirects) to verify the certificate status.
+    #
+    # @see https://ruby-doc.org/stdlib/libdoc/openssl/rdoc/OpenSSL/OCSP.html
+    #
+    # @api private
+    class OcspVerifier
+      include Loggable
+
+      # @param [ String ] host_name The host name being verified, for
+      #   diagnostic output.
+      # @param [ OpenSSL::X509::Certificate ] cert The certificate presented by
+      #   the server at host_name.
+      # @param [ OpenSSL::X509::Certificate ] ca_cert The CA certificate
+      #   presented by the server or resolved locally from the server
+      #   certificate.
+      # @param [ OpenSSL::X509::Store ] cert_store The certificate store to
+      #   use for verifying OCSP response. This should be the same store as
+      #   used in SSLContext used with the SSLSocket that we are verifying the
+      #   certificate for. This must NOT be the CA certificate provided by
+      #   the server (i.e. anything taken out of peer_cert) - otherwise the
+      #   server would dictate which CA authorities the client trusts.
+      def initialize(host_name, cert, ca_cert, cert_store, **opts)
+        @host_name = host_name
+        @cert = cert
+        @ca_cert = ca_cert
+        @cert_store = cert_store
+        @options = opts
+      end
+
+      attr_reader :host_name
+      attr_reader :cert
+      attr_reader :ca_cert
+      attr_reader :cert_store
+      attr_reader :options
+
+      def timeout
+        options[:timeout] || 5
+      end
+
+      # @return [ Array<String> ] OCSP URIs in the specified server certificate.
+      def ocsp_uris
+        @ocsp_uris ||= begin
+          # https://tools.ietf.org/html/rfc3546#section-2.3
+          # prohibits multiple extensions with the same oid.
+          ext = cert.extensions.detect do |ext|
+            ext.oid == 'authorityInfoAccess'
+          end
+
+          if ext
+            # Our test certificates have multiple OCSP URIs.
+            ext.value.split("\n").select do |line|
+              line.start_with?('OCSP - URI:')
+            end.map do |line|
+              line.split(':', 2).last
+            end
+          else
+            []
+          end
+        end
+      end
+
+      def cert_id
+        @cert_id ||= OpenSSL::OCSP::CertificateId.new(
+          cert,
+          ca_cert,
+          OpenSSL::Digest::SHA1.new,
+        )
+      end
+
+      def verify_with_cache
+        handle_exceptions do
+          return false if ocsp_uris.empty?
+
+          resp = OcspCache.get(cert_id)
+          if resp
+            return return_ocsp_response(resp)
+          end
+
+          resp, errors = do_verify
+
+          if resp
+            OcspCache.set(cert_id, resp)
+          end
+
+          return_ocsp_response(resp, errors)
+        end
+      end
+
+      # @return [ true | false ] Whether the certificate was verified.
+      #
+      # @raise [ Error::ServerCertificateRevoked ] If the certificate was
+      #   definitively revoked.
+      def verify
+        handle_exceptions do
+          return false if ocsp_uris.empty?
+
+          resp, errors = do_verify
+          return_ocsp_response(resp, errors)
+        end
+      end
+
+      private
+
+      def do_verify
+        # This synchronized array contains definitive pass/fail responses
+        # obtained from the responders. We'll take the first one but due to
+        # concurrency multiple responses may be produced and queued.
+        @resp_queue = Queue.new
+
+        # This synchronized array contains strings, one per responder, that
+        # explain why each responder hasn't produced a definitive response.
+        # These are concatenated and logged if none of the responders produced
+        # a definitive respnose, or if the main thread times out waiting for
+        # a definitive response (in which case some of the worker threads'
+        # diagnostics may be logged and some may not).
+        @resp_errors = Queue.new
+
+        @req = OpenSSL::OCSP::Request.new
+        @req.add_certid(cert_id)
+        @req.add_nonce
+        @serialized_req = @req.to_der
+
+        @outstanding_requests = ocsp_uris.count
+        @outstanding_requests_lock = Mutex.new
+
+        threads = ocsp_uris.map do |uri|
+          Thread.new do
+            verify_one_responder(uri)
+          end
+        end
+
+        resp = begin
+          ::Timeout.timeout(timeout) do
+            @resp_queue.shift
+          end
+        rescue ::Timeout::Error
+          nil
+        end
+
+        threads.map(&:kill)
+        threads.map(&:join)
+
+        [resp, @resp_errors]
+      end
+
+      def verify_one_responder(uri)
+        original_uri = uri
+        redirect_count = 0
+        http_response = nil
+        loop do
+          http_response = begin
+            uri = URI(uri)
+            Net::HTTP.start(uri.hostname, uri.port) do |http|
+              path = uri.path
+              if path.empty?
+                path = '/'
+              end
+              http.post(path, @serialized_req,
+                'content-type' => 'application/ocsp-request')
+            end
+          rescue IOError, SystemCallError => e
+            @resp_errors << "OCSP request to #{report_uri(original_uri, uri)} failed: #{e.class}: #{e}"
+            return false
+          end
+
+          code = http_response.code.to_i
+          if (300..399).include?(code)
+            redirected_uri = http_response.header['location']
+            uri = ::URI.join(uri, redirected_uri)
+            redirect_count += 1
+            if redirect_count > 5
+              @resp_errors << "OCSP request to #{report_uri(original_uri, uri)} failed: too many redirects (6)"
+              return false
+            end
+            next
+          end
+
+          if code >= 400
+            @resp_errors << "OCSP request to #{report_uri(original_uri, uri)} failed with HTTP status code #{http_response.code}" + report_response_body(http_response.body)
+            return false
+          end
+
+          if code != 200
+            # There must be a body provided with the response, if one isn't
+            # provided the response cannot be verified.
+            @resp_errors << "OCSP request to #{report_uri(original_uri, uri)} failed with unexpected HTTP status code #{http_response.code}" + report_response_body(http_response.body)
+            return false
+          end
+
+          break
+        end
+
+        resp = OpenSSL::OCSP::Response.new(http_response.body).basic
+        unless resp.verify([ca_cert], cert_store)
+          # Ruby's OpenSSL binding discards error information - see
+          # https://github.com/ruby/openssl/issues/395
+          @resp_errors << "OCSP response from #{report_uri(original_uri, uri)} failed signature verification; set `OpenSSL.debug = true` to see why"
+          return false
+        end
+
+        if @req.check_nonce(resp) == 0
+          @resp_errors << "OCSP response from #{report_uri(original_uri, uri)} included invalid nonce"
+          return false
+        end
+
+        if resp.respond_to?(:find_response)
+          # Ruby 2.4+
+          resp = resp.find_response(cert_id)
+          # TODO make a new class instead of patching the stdlib one?
+          resp.instance_variable_set('@uri', uri)
+          resp.instance_variable_set('@original_uri', original_uri)
+          class << resp
+            attr_reader :uri, :original_uri
+          end
+        else
+          # Ruby 2.3
+          found = nil
+          resp.status.each do |_cert_id, cert_status, revocation_reason, revocation_time, this_update, next_update, extensions|
+            if _cert_id.cmp(cert_id)
+              found = OpenStruct.new(
+                cert_status: cert_status,
+                certid: _cert_id,
+                next_update: next_update,
+                this_update: this_update,
+                revocation_reason: revocation_reason,
+                revocation_time: revocation_time,
+                extensions: extensions,
+                uri: uri,
+                original_uri: original_uri,
+              )
+              class << found
+                # Unlike the stdlib method, this one doesn't accept
+                # any arguments.
+                def check_validity
+                  now = Time.now
+                  this_update <= now && next_update >= now
+                end
+              end
+              break
+            end
+          end
+          resp = found
+        end
+
+        unless resp
+          @resp_errors << "OCSP response from #{report_uri(original_uri, uri)} did not include information about the requested certificate"
+          return false
+        end
+
+        unless resp.check_validity
+          @resp_errors << "OCSP response from #{report_uri(original_uri, uri)} was invalid: this_update was in the future or next_update time has passed"
+          return false
+        end
+
+        unless [
+          OpenSSL::OCSP::V_CERTSTATUS_GOOD,
+          OpenSSL::OCSP::V_CERTSTATUS_REVOKED,
+        ].include?(resp.cert_status)
+          @resp_errors << "OCSP response from #{report_uri(original_uri, uri)} had a non-definitive status: #{resp.cert_status}"
+          return false
+        end
+
+        # Note this returns the redirected URI
+        @resp_queue << resp
+      rescue => exc
+        Utils.warn_bg_exception("Error performing OCSP verification for '#{host_name}' via '#{uri}'", exc,
+          logger: options[:logger],
+          log_prefix: options[:log_prefix],
+          bg_error_backtrace: options[:bg_error_backtrace],
+        )
+        false
+      ensure
+        @outstanding_requests_lock.synchronize do
+          @outstanding_requests -= 1
+          if @outstanding_requests == 0
+            @resp_queue << nil
+          end
+        end
+      end
+
+      def return_ocsp_response(resp, errors = nil)
+        if resp
+          if resp.cert_status == OpenSSL::OCSP::V_CERTSTATUS_REVOKED
+            raise_revoked_error(resp)
+          end
+          true
+        else
+          reasons = []
+          errors.length.times do
+            reasons << errors.shift
+          end
+          if reasons.empty?
+            msg = "No responses from responders: #{ocsp_uris.join(', ')} within #{timeout} seconds"
+          else
+            msg = "For responders #{ocsp_uris.join(', ')} with a timeout of #{timeout} seconds: #{reasons.join(', ')}"
+          end
+          log_warn("TLS certificate of '#{host_name}' could not be definitively verified via OCSP: #{msg}")
+          false
+        end
+      end
+
+      def handle_exceptions
+        begin
+          yield
+        rescue Error::ServerCertificateRevoked
+          raise
+        rescue => exc
+          Utils.warn_bg_exception(
+            "Error performing OCSP verification for '#{host_name}'",
+            exc,
+            **options)
+          false
+        end
+      end
+
+      def raise_revoked_error(resp)
+        if resp.uri == resp.original_uri
+          redirect = ''
+        else
+          redirect = " (redirected from #{resp.original_uri})"
+        end
+        raise Error::ServerCertificateRevoked, "TLS certificate of '#{host_name}' has been revoked according to '#{resp.uri}'#{redirect} for reason '#{resp.revocation_reason}' at '#{resp.revocation_time}'"
+      end
+
+      def report_uri(original_uri, uri)
+        if URI(uri) == URI(original_uri)
+          uri
+        else
+          "#{original_uri} (redirected to #{uri})"
+        end
+      end
+
+      def report_response_body(body)
+        if body
+          ": #{body}"
+        else
+          ''
+        end
+      end
+    end
+  end
+end