mongo 2.4.1 → 2.14.0

data/lib/mongo/server/monitor.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016 MongoDB Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,100 +12,111 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'mongo/server/monitor/connection'
-
 module Mongo
   class Server
 
-    # This object is responsible for keeping server status up to date, running in
-    # a separate thread as to not disrupt other operations.
+    # Responsible for periodically polling a server via ismaster commands to
+    # keep the server's status up to date.
+    #
+    # Does all work in a background thread so as to not interfere with other
+    # operations performed by the driver.
     #
     # @since 2.0.0
+    # @api private
     class Monitor
       include Loggable
+      extend Forwardable
+      include Event::Publisher
+      include BackgroundThread
 
-      # The default time for a server to refresh its status is 10 seconds.
+      # The default interval between server status refreshes is 10 seconds.
       #
       # @since 2.0.0
-      HEARTBEAT_FREQUENCY = 10.freeze
+      DEFAULT_HEARTBEAT_INTERVAL = 10.freeze
 
       # The minimum time between forced server scans. Is
       # minHeartbeatFrequencyMS in the SDAM spec.
       #
       # @since 2.0.0
-      MIN_SCAN_FREQUENCY = 0.5.freeze
+      MIN_SCAN_INTERVAL = 0.5.freeze
 
       # The weighting factor (alpha) for calculating the average moving round trip time.
       #
       # @since 2.0.0
+      # @deprecated Will be removed in version 3.0.
       RTT_WEIGHT_FACTOR = 0.2.freeze
 
-      # @return [ Mongo::Connection ] connection The connection to use.
-      attr_reader :connection
-
-      # @return [ Server::Description ] description The server
-      #   description the monitor refreshes.
-      attr_reader :description
-
-      # @return [ Description::Inspector ] inspector The description inspector.
-      attr_reader :inspector
-
-      # @return [ Hash ] options The server options.
-      attr_reader :options
-
-      # @return [ Time ] last_scan The time of the last server scan.
-      #
-      # @since 2.4.0
-      attr_reader :last_scan
-
-      # Force the monitor to immediately do a check of its server.
-      #
-      # @example Force a scan.
-      #   monitor.scan!
-      #
-      # @return [ Description ] The updated description.
-      #
-      # @since 2.0.0
-      def scan!
-        throttle_scan_frequency!
-        @description = inspector.run(description, *ismaster)
-      end
-
-      # Get the refresh interval for the server. This will be defined via an option
-      # or will default to 5.
-      #
-      # @example Get the refresh interval.
-      #   server.heartbeat_frequency
-      #
-      # @return [ Integer ] The heartbeat frequency, in seconds.
-      #
-      # @since 2.0.0
-      def heartbeat_frequency
-        @heartbeat_frequency ||= options[:heartbeat_frequency] || HEARTBEAT_FREQUENCY
-      end
-
       # Create the new server monitor.
       #
-      # @api private
-      #
       # @example Create the server monitor.
-      #   Mongo::Server::Monitor.new(address, listeners)
+      #   Mongo::Server::Monitor.new(address, listeners, monitoring)
       #
       # @note Monitor must never be directly instantiated outside of a Server.
       #
-      # @param [ Address ] address The address to monitor.
-      # @param [ Event::Listeners ] listeners The event listeners.
+      # @param [ Server ] server The server to monitor.
+      # @param [ Event::Listeners ] event_listeners The event listeners.
+      # @param [ Monitoring ] monitoring The monitoring..
       # @param [ Hash ] options The options.
       #
+      # @option options [ Float ] :connect_timeout The timeout, in seconds, to
+      #   use when establishing the monitoring connection.
+      # @option options [ Float ] :heartbeat_interval The interval between
+      #   regular server checks.
+      # @option options [ Logger ] :logger A custom logger to use.
+      # @option options [ Float ] :socket_timeout The timeout, in seconds, to
+      #   execute operations on the monitoring connection.
+      #
       # @since 2.0.0
-      def initialize(address, listeners, options = {})
-        @description = Description.new(address, {})
-        @inspector = Description::Inspector.new(listeners)
+      # @api private
+      def initialize(server, event_listeners, monitoring, options = {})
+        unless monitoring.is_a?(Monitoring)
+          raise ArgumentError, "Wrong monitoring type: #{monitoring.inspect}"
+        end
+        @server = server
+        @event_listeners = event_listeners
+        @monitoring = monitoring
         @options = options.freeze
-        @connection = Connection.new(address, options)
-        @last_round_trip_time = nil
-        @last_scan = nil
         @mutex = Mutex.new
+        @sdam_mutex = Mutex.new
+        @next_earliest_scan = @next_wanted_scan = Time.now
+        @update_mutex = Mutex.new
+      end
+
+      # @return [ Server ] server The server that this monitor is monitoring.
+      # @api private
+      attr_reader :server
+
+      # @return [ Mongo::Server::Monitor::Connection ] connection The connection to use.
+      attr_reader :connection
+
+      # @return [ Hash ] options The server options.
+      attr_reader :options
+
+      # The interval between regular server checks.
+      #
+      # @return [ Float ] The heartbeat interval, in seconds.
+      def heartbeat_interval
+        options[:heartbeat_interval] || DEFAULT_HEARTBEAT_INTERVAL
+      end
+
+      # @deprecated
+      def_delegators :server, :last_scan
+
+      # The compressor is determined during the handshake, so it must be an
+      # attribute of the connection.
+      #
+      # @deprecated
+      def_delegators :connection, :compressor
+
+      # @return [ Monitoring ] monitoring The monitoring.
+      attr_reader :monitoring
+
+      # @return [ Server::PushMonitor | nil ] The push monitor, if one is being
+      #   used.
+      def push_monitor
+        @update_mutex.synchronize do
+          @push_monitor
+        end
       end
 
       # Runs the server monitor. Refreshing happens on a separate thread per
@@ -117,26 +128,101 @@ module Mongo
       # @return [ Thread ] The thread the monitor runs on.
       #
       # @since 2.0.0
-      def run!
-        @thread = Thread.new(heartbeat_frequency) do |i|
-          loop do
-            sleep(i)
-            scan!
+      def do_work
+        scan!
+        # @next_wanted_scan may be updated by the push monitor.
+        # However we need to check for termination flag so that the monitor
+        # thread exits when requested.
+        loop do
+          delta = @next_wanted_scan - Time.now
+          if delta > 0
+            signaled = server.scan_semaphore.wait(delta)
+            if signaled || @stop_requested
+              break
+            end
+          else
+            break
           end
         end
       end
 
-      # Stops the server monitor. Kills the thread so it doesn't continue
-      # taking memory and sending commands to the connection.
+      # Stop the background thread and wait for it to terminate for a
+      # reasonable amount of time.
       #
-      # @example Stop the monitor.
-      #   monitor.stop!
+      # @return [ true | false ] Whether the thread was terminated.
       #
-      # @return [ Boolean ] Is the Thread stopped?
+      # @api public for backwards compatibility only
+      def stop!
+        stop_push_monitor!
+
+        # Forward super's return value
+        super.tap do
+          # Important: disconnect should happen after the background thread
+          # terminates.
+          connection&.disconnect!
+        end
+      end
+
+      def stop_push_monitor!
+        @update_mutex.synchronize do
+          if @push_monitor
+            @push_monitor.stop!
+            @push_monitor = nil
+          end
+        end
+      end
+
+      # Perform a check of the server with throttling, and update
+      # the server's description and average round trip time.
+      #
+      # If the server was checked less than MIN_SCAN_INTERVAL seconds
+      # ago, sleep until MIN_SCAN_INTERVAL seconds have passed since the last
+      # check. Then perform the check which involves running isMaster
+      # on the server being monitored and updating the server description
+      # as a result.
+      #
+      # @note If the system clock moves backwards, this method can sleep
+      #   for a very long time.
+      #
+      # @note The return value of this method is deprecated. In version 3.0.0
+      #   this method will not have a return value.
+      #
+      # @return [ Description ] The updated description.
       #
       # @since 2.0.0
-      def stop!
-        connection.disconnect! && @thread.kill && @thread.stop?
+      def scan!
+        # Ordinarily the background thread would invoke this method.
+        # But it is also possible to invoke scan! directly on a monitor.
+        # Allow only one scan to be performed at a time.
+        @mutex.synchronize do
+          throttle_scan_frequency!
+
+          result = do_scan
+
+          run_sdam_flow(result)
+        end
+      end
+
+      def run_sdam_flow(result, awaited: false)
+        @sdam_mutex.synchronize do
+          old_description = server.description
+
+          new_description = Description.new(server.address, result,
+            server.round_trip_time_averager.average_round_trip_time)
+
+          server.cluster.run_sdam_flow(server.description, new_description, awaited: awaited)
+
+          server.description.tap do |new_description|
+            unless awaited
+              if new_description.unknown? && !old_description.unknown?
+                @next_earliest_scan = @next_wanted_scan = Time.now
+              else
+                @next_earliest_scan = Time.now + MIN_SCAN_INTERVAL
+                @next_wanted_scan = Time.now + heartbeat_interval
+              end
+            end
+          end
+        end
       end
 
       # Restarts the server monitor unless the current thread is alive.
@@ -148,40 +234,93 @@ module Mongo
       #
       # @since 2.1.0
       def restart!
-        @thread.alive? ? @thread : run!
+        if @thread && @thread.alive?
+          @thread
+        else
+          run!
+        end
       end
 
       private
 
-      def average_round_trip_time(start)
-        new_rtt = Time.now - start
-        RTT_WEIGHT_FACTOR * new_rtt + (1 - RTT_WEIGHT_FACTOR) * (@last_round_trip_time || new_rtt)
+      def pre_stop
+        server.scan_semaphore.signal
       end
 
-      def calculate_average_round_trip_time(start)
-        @last_round_trip_time = average_round_trip_time(start)
+      def do_scan
+        begin
+          monitoring.publish_heartbeat(server) do
+            ismaster
+          end
+        rescue => exc
+          msg = "Error running ismaster on #{server.address}"
+          Utils.warn_bg_exception(msg, exc,
+            logger: options[:logger],
+            log_prefix: options[:log_prefix],
+            bg_error_backtrace: options[:bg_error_backtrace],
+          )
+          {}
+        end
       end
 
       def ismaster
-        @mutex.synchronize do
-          start = Time.now
-          begin
-            return connection.ismaster, calculate_average_round_trip_time(start)
-          rescue Exception => e
-            log_debug(e.message)
-            return {}, calculate_average_round_trip_time(start)
+        if @connection && @connection.pid != Process.pid
+          log_warn("Detected PID change - Mongo client should have been reconnected (old pid #{@connection.pid}, new pid #{Process.pid}")
+          @connection.disconnect!
+          @connection = nil
+        end
+
+        if @connection
+          result = server.round_trip_time_averager.measure do
+            begin
+              message = @connection.dispatch_bytes(Monitor::Connection::ISMASTER_BYTES)
+              message.documents.first
+            rescue Mongo::Error
+              @connection.disconnect!
+              @connection = nil
+              raise
+            end
+          end
+        else
+          connection = Connection.new(server.address, options)
+          connection.connect!
+          result = server.round_trip_time_averager.measure do
+            connection.handshake!
+          end
+          @connection = connection
+          if result['topologyVersion']
+            # Successful response, server 4.4+
+            @update_mutex.synchronize do
+              @push_monitor ||= PushMonitor.new(
+                self,
+                TopologyVersion.new(result['topologyVersion']),
+                monitoring,
+                **Utils.shallow_symbolize_keys(options.merge(
+                  socket_timeout: heartbeat_interval + connection.socket_timeout,
+                )),
+              )
+            end
+            push_monitor.run!
+          else
+            # Failed response or pre-4.4 server
+            stop_push_monitor!
           end
+          result
         end
+        result
       end
 
+      # @note If the system clock is set to a time in the past, this method
+      #   can sleep for a very long time.
       def throttle_scan_frequency!
-        if @last_scan
-          difference = (Time.now - @last_scan)
-          throttle_time = (MIN_SCAN_FREQUENCY - difference)
-          sleep(throttle_time) if throttle_time > 0
+        delta = @next_earliest_scan - Time.now
+        if delta > 0
+          sleep(delta)
         end
-        @last_scan = Time.now
       end
     end
   end
 end
+
+require 'mongo/server/monitor/connection'
+require 'mongo/server/monitor/app_metadata'

data/lib/mongo/server/pending_connection.rb

@@ -0,0 +1,255 @@
+# Copyright (C) 2019-2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Server
+
+    # This class encapsulates connections during handshake and authentication.
+    #
+    # @api private
+    class PendingConnection < ConnectionBase
+      extend Forwardable
+
+      def initialize(socket, server, monitoring, options = {})
+        @socket = socket
+        @options = options
+        @server = server
+        @monitoring = monitoring
+        @id = options[:id]
+      end
+
+      # @return [ Integer ] The ID for the connection. This is the same ID
+      #   as that of the regular Connection object for which this
+      #   PendingConnection instance was created.
+      attr_reader :id
+
+      def handshake_and_authenticate!
+        speculative_auth_doc = nil
+        if options[:user] || options[:auth_mech]
+          # To create an Auth instance, we need to specify the mechanism,
+          # but at this point we don't know the mechanism that ultimately
+          # will be used (since this depends on the data returned by
+          # the handshake, specifically server version).
+          # However, we know that only 4.4+ servers support speculative
+          # authentication, and those servers also generally support
+          # SCRAM-SHA-256. We expect that user accounts created for 4.4+
+          # servers would generally allow SCRAM-SHA-256 authentication;
+          # user accounts migrated from pre-4.4 servers may only allow
+          # SCRAM-SHA-1. The use of SCRAM-SHA-256 by default is thus
+          # sensible, and it is also mandated by the speculative auth spec.
+          # If no mechanism was specified and we are talking to a 3.0+
+          # server, we'll send speculative auth document, the server will
+          # ignore it and we'll perform authentication using explicit
+          # command after having defaulted the mechanism later to CR.
+          # If no mechanism was specified and we are talking to a 4.4+
+          # server and the user account doesn't allow SCRAM-SHA-256, we will
+          # authenticate in a separate command with SCRAM-SHA-1 after
+          # going through SCRAM mechanism negotiation.
+          default_options = Options::Redacted.new(:auth_mech => :scram256)
+          speculative_auth_user = Auth::User.new(default_options.merge(options))
+          speculative_auth = Auth.get(speculative_auth_user, self)
+          speculative_auth_doc = speculative_auth.conversation.speculative_auth_document
+        end
+
+        result = handshake!(speculative_auth_doc: speculative_auth_doc)
+        if speculative_auth_doc && (speculative_auth_result = result['speculativeAuthenticate'])
+          unless description.features.scram_sha_1_enabled?
+            raise Error::InvalidServerAuthResponse, "Speculative auth succeeded on a pre-3.0 server"
+          end
+          case speculative_auth_user.mechanism
+          when :mongodb_x509
+            # Done
+          # We default auth mechanism to scram256, but if user specified
+          # scram explicitly we may be able to authenticate speculatively
+          # with scram.
+          when :scram, :scram256
+            authenticate!(
+              speculative_auth_client_nonce: speculative_auth.conversation.client_nonce,
+              speculative_auth_mech: speculative_auth_user.mechanism,
+              speculative_auth_result: speculative_auth_result,
+            )
+          else
+            raise NotImplementedError, "Speculative auth unexpectedly succeeded for mechanism #{speculative_auth_user.mechanism.inspect}"
+          end
+        elsif !description.arbiter?
+          authenticate!
+        end
+      end
+
+      private
+
+      # @param [ BSON::Document | nil ] speculative_auth_doc The document to
+      #   provide in speculativeAuthenticate field of ismaster command.
+      #
+      # @return [ BSON::Document ] The document of the ismaster response for
+      #   this particular connection.
+      def handshake!(speculative_auth_doc: nil)
+        unless socket
+          raise Error::HandshakeError, "Cannot handshake because there is no usable socket (for #{address})"
+        end
+
+        ismaster_doc = app_metadata.validated_document
+        if speculative_auth_doc
+          ismaster_doc = ismaster_doc.merge(speculativeAuthenticate: speculative_auth_doc)
+        end
+
+        ismaster_command = Protocol::Query.new(Database::ADMIN, Database::COMMAND,
+          ismaster_doc, :limit => -1)
+
+        response = nil
+        @server.handle_handshake_failure! do
+          begin
+            response = @server.round_trip_time_averager.measure do
+              add_server_diagnostics do
+                socket.write(ismaster_command.serialize.to_s)
+                Protocol::Message.deserialize(socket, Protocol::Message::MAX_MESSAGE_SIZE).documents.first
+              end
+            end
+          rescue => exc
+            msg = "Failed to handshake with #{address}"
+            Utils.warn_bg_exception(msg, exc,
+              logger: options[:logger],
+              log_prefix: options[:log_prefix],
+              bg_error_backtrace: options[:bg_error_backtrace],
+            )
+            raise
+          end
+        end
+
+        post_handshake(response, @server.round_trip_time_averager.average_round_trip_time)
+
+        response
+      end
+
+      # @param [ String | nil ] speculative_auth_client_nonce The client
+      #   nonce used in speculative auth on this connection that
+      #   produced the specified speculative auth result.
+      # @param [ Symbol | nil ] speculative_auth_mech Auth mechanism used
+      #   for speculative auth, if speculative auth succeeded. If speculative
+      #   auth was not performed or it failed, this must be nil.
+      # @param [ BSON::Document | nil ] speculative_auth_result The
+      #   value of speculativeAuthenticate field of ismaster response of
+      #   the handshake on this connection.
+      def authenticate!(
+        speculative_auth_client_nonce: nil,
+        speculative_auth_mech: nil,
+        speculative_auth_result: nil
+      )
+        if options[:user] || options[:auth_mech]
+          @server.handle_auth_failure! do
+            begin
+              auth = Auth.get(
+                resolved_user(speculative_auth_mech: speculative_auth_mech),
+                self,
+                speculative_auth_client_nonce: speculative_auth_client_nonce,
+                speculative_auth_result: speculative_auth_result,
+              )
+              auth.login
+            rescue => exc
+              msg = "Failed to authenticate to #{address}"
+              Utils.warn_bg_exception(msg, exc,
+                logger: options[:logger],
+                log_prefix: options[:log_prefix],
+                bg_error_backtrace: options[:bg_error_backtrace],
+              )
+              raise
+            end
+          end
+        end
+      end
+
+      def ensure_connected
+        yield @socket
+      end
+
+      # This is a separate method to keep the nesting level down.
+      #
+      # @return [ Server::Description ] The server description calculated from
+      #   ismaster response for this particular connection.
+      def post_handshake(response, average_rtt)
+        if response["ok"] == 1
+          # Auth mechanism is entirely dependent on the contents of
+          # ismaster response *for this connection*.
+          # Ismaster received by the monitoring connection should advertise
+          # the same wire protocol, but if it doesn't, we use whatever
+          # the monitoring connection advertised for filling out the
+          # server description and whatever the non-monitoring connection
+          # (that's this one) advertised for performing auth on that
+          # connection.
+          @sasl_supported_mechanisms = response['saslSupportedMechs']
+          set_compressor!(response)
+        else
+          @sasl_supported_mechanisms = nil
+        end
+
+        @description = Description.new(address, response, average_rtt).tap do |new_description|
+          @server.cluster.run_sdam_flow(@server.description, new_description)
+        end
+      end
+
+      # The user as going to be used for authentication. This user has the
+      # auth mechanism set and, if necessary, auth source.
+      #
+      # @param [ Symbol | nil ] speculative_auth_mech Auth mechanism used
+      #   for speculative auth, if speculative auth succeeded. If speculative
+      #   auth was not performed or it failed, this must be nil.
+      #
+      # @return [ Auth::User ] The resolved user.
+      def resolved_user(speculative_auth_mech: nil)
+        @resolved_user ||= begin
+          unless options[:user] || options[:auth_mech]
+            raise Mongo::Error, 'No authentication information specified in the client'
+          end
+
+          user_options = Options::Redacted.new(
+            # When speculative auth is performed, we always use SCRAM-SHA-256.
+            # At the same time we perform SCRAM mechanism negotiation in the
+            # ismaster request.
+            # If the credentials we are trying to authenticate with do not
+            # map to an existing user, SCRAM mechanism negotiation will not
+            # return anything which would cause the driver to use
+            # SCRAM-SHA-1. However, on 4.4+ servers speculative auth would
+            # succeed (technically just the first round-trip, not the entire
+            # authentication flow) and we would be continuing it here;
+            # in this case, we must use SCRAM-SHA-256 as the mechanism since
+            # that is what the conversation was started with, even though
+            # SCRAM mechanism negotiation did not return SCRAM-SHA-256 as a
+            # valid mechanism to use for these credentials.
+            :auth_mech => speculative_auth_mech || default_mechanism,
+          ).merge(options)
+          if user_options[:auth_mech] == :mongodb_x509
+            user_options[:auth_source] = '$external'
+          end
+          Auth::User.new(user_options)
+        end
+      end
+
+      def default_mechanism
+        if description.nil?
+          raise Mongo::Error, 'Trying to query default mechanism when handshake has not completed'
+        end
+
+        if description.features.scram_sha_1_enabled?
+          if @sasl_supported_mechanisms&.include?('SCRAM-SHA-256')
+            :scram256
+          else
+            :scram
+          end
+        else
+          :mongodb_cr
+        end
+      end
+    end
+  end
+end

data/lib/mongo/server/push_monitor/connection.rb

@@ -0,0 +1,28 @@
+# Copyright (C) 2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Server
+    class PushMonitor
+
+      # @api private
+      class Connection < Server::Monitor::Connection
+
+        def socket_timeout
+          options[:socket_timeout]
+        end
+      end
+    end
+  end
+end