mongo 2.11.4 → 2.18.1

data/lib/mongo/server/monitor.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2019 MongoDB Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,29 +18,30 @@
 module Mongo
   class Server
 
-    # Responsible for periodically polling a server via ismaster commands to
+    # Responsible for periodically polling a server via hello commands to
     # keep the server's status up to date.
     #
     # Does all work in a background thread so as to not interfere with other
     # operations performed by the driver.
     #
     # @since 2.0.0
+    # @api private
     class Monitor
       include Loggable
       extend Forwardable
       include Event::Publisher
       include BackgroundThread
 
-      # The default time for a server to refresh its status is 10 seconds.
+      # The default interval between server status refreshes is 10 seconds.
       #
       # @since 2.0.0
-      HEARTBEAT_FREQUENCY = 10.freeze
+      DEFAULT_HEARTBEAT_INTERVAL = 10.freeze
 
       # The minimum time between forced server scans. Is
       # minHeartbeatFrequencyMS in the SDAM spec.
       #
       # @since 2.0.0
-      MIN_SCAN_FREQUENCY = 0.5.freeze
+      MIN_SCAN_INTERVAL = 0.5.freeze
 
       # The weighting factor (alpha) for calculating the average moving round trip time.
       #
@@ -59,7 +63,13 @@ module Mongo
       #
       # @option options [ Float ] :connect_timeout The timeout, in seconds, to
       #   use when establishing the monitoring connection.
+      # @option options [ Float ] :heartbeat_interval The interval between
+      #   regular server checks.
       # @option options [ Logger ] :logger A custom logger to use.
+      # @option options [ Mongo::Server::Monitor::AppMetadata ] :monitor_app_metadata
+      #   The metadata to use for regular monitoring connection.
+      # @option options [ Mongo::Server::Monitor::AppMetadata ] :push_monitor_app_metadata
+      #   The metadata to use for push monitor's connection.
       # @option options [ Float ] :socket_timeout The timeout, in seconds, to
       #   execute operations on the monitoring connection.
       #
@@ -69,14 +79,20 @@ module Mongo
         unless monitoring.is_a?(Monitoring)
           raise ArgumentError, "Wrong monitoring type: #{monitoring.inspect}"
         end
+        unless options[:app_metadata]
+          raise ArgumentError, 'App metadata is required'
+        end
+        unless options[:push_monitor_app_metadata]
+          raise ArgumentError, 'Push monitor app metadata is required'
+        end
         @server = server
         @event_listeners = event_listeners
         @monitoring = monitoring
         @options = options.freeze
-        # This is a Mongo::Server::Monitor::Connection
-        @connection = Connection.new(server.address, options)
         @mutex = Mutex.new
-        @scan_started_at = nil
+        @sdam_mutex = Mutex.new
+        @next_earliest_scan = @next_wanted_scan = Time.now
+        @update_mutex = Mutex.new
       end
 
       # @return [ Server ] server The server that this monitor is monitoring.
@@ -89,84 +105,151 @@ module Mongo
       # @return [ Hash ] options The server options.
       attr_reader :options
 
+      # The interval between regular server checks.
+      #
+      # @return [ Float ] The heartbeat interval, in seconds.
+      def heartbeat_interval
+        options[:heartbeat_interval] || DEFAULT_HEARTBEAT_INTERVAL
+      end
+
       # @deprecated
       def_delegators :server, :last_scan
 
-      # The compressor is determined during the handshake, so it must be an attribute
-      # of the connection.
+      # The compressor is determined during the handshake, so it must be an
+      # attribute of the connection.
+      #
+      # @deprecated
       def_delegators :connection, :compressor
 
       # @return [ Monitoring ] monitoring The monitoring.
       attr_reader :monitoring
 
-      # Get the refresh interval for the server. This will be defined via an
-      # option or will default to 10.
-      #
-      # @return [ Float ] The heartbeat interval, in seconds.
-      #
-      # @since 2.0.0
-      # @deprecated
-      def heartbeat_frequency
-        server.cluster.heartbeat_interval
+      # @return [ Server::PushMonitor | nil ] The push monitor, if one is being
+      #   used.
+      def push_monitor
+        @update_mutex.synchronize do
+          @push_monitor
+        end
       end
 
-      # Runs the server monitor. Refreshing happens on a separate thread per
-      # server.
-      #
-      # @example Run the monitor.
-      #   monitor.run
-      #
-      # @return [ Thread ] The thread the monitor runs on.
+      # Perform a check of the server.
       #
       # @since 2.0.0
       def do_work
         scan!
-        server.scan_semaphore.wait(server.cluster.heartbeat_interval)
+        # @next_wanted_scan may be updated by the push monitor.
+        # However we need to check for termination flag so that the monitor
+        # thread exits when requested.
+        loop do
+          delta = @next_wanted_scan - Time.now
+          if delta > 0
+            signaled = server.scan_semaphore.wait(delta)
+            if signaled || @stop_requested
+              break
+            end
+          else
+            break
+          end
+        end
       end
 
-      # Stop the background thread and wait for to terminate for a reasonable
-      # amount of time.
+      # Stop the background thread and wait for it to terminate for a
+      # reasonable amount of time.
       #
       # @return [ true | false ] Whether the thread was terminated.
       #
       # @api public for backwards compatibility only
       def stop!
+        stop_push_monitor!
+
         # Forward super's return value
         super.tap do
           # Important: disconnect should happen after the background thread
-          # terminated.
-          connection.disconnect!
+          # terminates.
+          connection&.disconnect!
+        end
+      end
+
+      def create_push_monitor!(topology_version)
+        @update_mutex.synchronize do
+          if @push_monitor && !@push_monitor.running?
+            @push_monitor = nil
+          end
+
+          @push_monitor ||= PushMonitor.new(
+            self,
+            topology_version,
+            monitoring,
+            **Utils.shallow_symbolize_keys(options.merge(
+              socket_timeout: heartbeat_interval + connection.socket_timeout,
+              app_metadata: options[:push_monitor_app_metadata],
+              check_document: @connection.check_document
+            )),
+          )
+        end
+      end
+
+      def stop_push_monitor!
+        @update_mutex.synchronize do
+          if @push_monitor
+            @push_monitor.stop!
+            @push_monitor = nil
+          end
         end
       end
 
       # Perform a check of the server with throttling, and update
       # the server's description and average round trip time.
       #
-      # If the server was checked less than MIN_SCAN_FREQUENCY seconds
-      # ago, sleep until MIN_SCAN_FREQUENCY seconds have passed since the last
-      # check. Then perform the check which involves running isMaster
+      # If the server was checked less than MIN_SCAN_INTERVAL seconds
+      # ago, sleep until MIN_SCAN_INTERVAL seconds have passed since the last
+      # check. Then perform the check which involves running hello
       # on the server being monitored and updating the server description
       # as a result.
       #
-      # @note If the system clock is set to a time in the past, this method
-      #   can sleep for a very long time.
+      # @note If the system clock moves backwards, this method can sleep
+      #   for a very long time.
       #
       # @note The return value of this method is deprecated. In version 3.0.0
       #   this method will not have a return value.
       #
-      # @example Run a scan.
-      #   monitor.scan!
-      #
       # @return [ Description ] The updated description.
       #
       # @since 2.0.0
       def scan!
-        throttle_scan_frequency!
-        result = ismaster
-        new_description = Description.new(server.address, result,
-          server.round_trip_time_averager.average_round_trip_time)
-        server.cluster.run_sdam_flow(server.description, new_description)
-        server.description
+        # Ordinarily the background thread would invoke this method.
+        # But it is also possible to invoke scan! directly on a monitor.
+        # Allow only one scan to be performed at a time.
+        @mutex.synchronize do
+          throttle_scan_frequency!
+
+          result = do_scan
+
+          run_sdam_flow(result)
+        end
+      end
+
+      def run_sdam_flow(result, awaited: false)
+        @sdam_mutex.synchronize do
+          old_description = server.description
+
+          new_description = Description.new(server.address, result,
+            average_round_trip_time: server.round_trip_time_averager.average_round_trip_time
+          )
+
+          server.cluster.run_sdam_flow(server.description, new_description, awaited: awaited)
+
+          server.description.tap do |new_description|
+            unless awaited
+              if new_description.unknown? && !old_description.unknown?
+                @next_earliest_scan = @next_wanted_scan = Time.now
+              else
+                @next_earliest_scan = Time.now + MIN_SCAN_INTERVAL
+                @next_wanted_scan = Time.now + heartbeat_interval
+              end
+            end
+          end
+        end
       end
 
       # Restarts the server monitor unless the current thread is alive.
@@ -185,60 +268,81 @@ module Mongo
         end
       end
 
+      def to_s
+        "#<#{self.class.name}:#{object_id} #{server.address}>"
+      end
+
       private
 
       def pre_stop
         server.scan_semaphore.signal
       end
 
-      def ismaster
-        @mutex.synchronize do
-          if monitoring.monitoring?
-            monitoring.started(
-              Monitoring::SERVER_HEARTBEAT,
-              Monitoring::Event::ServerHeartbeatStarted.new(server.address)
-            )
+      def do_scan
+        begin
+          monitoring.publish_heartbeat(server) do
+            check
           end
+        rescue => exc
+          msg = "Error checking #{server.address}"
+          Utils.warn_bg_exception(msg, exc,
+            logger: options[:logger],
+            log_prefix: options[:log_prefix],
+            bg_error_backtrace: options[:bg_error_backtrace],
+          )
+          {}
+        end
+      end
 
-          result, exc, rtt, average_rtt = server.round_trip_time_averager.measure do
-            connection.ismaster
-          end
-          if exc
-            log_debug("Error running ismaster on #{server.address}: #{exc.class}: #{exc}:\n#{exc.backtrace[0..5].join("\n")}")
-            if monitoring.monitoring?
-              monitoring.failed(
-                Monitoring::SERVER_HEARTBEAT,
-                Monitoring::Event::ServerHeartbeatFailed.new(server.address, rtt, exc)
+      def check
+        if @connection && @connection.pid != Process.pid
+          log_warn("Detected PID change - Mongo client should have been reconnected (old pid #{@connection.pid}, new pid #{Process.pid}")
+          @connection.disconnect!
+          @connection = nil
+        end
+
+        if @connection
+          result = server.round_trip_time_averager.measure do
+            begin
+              doc = @connection.check_document
+              cmd = Protocol::Query.new(
+                Database::ADMIN, Database::COMMAND, doc, :limit => -1
               )
+              message = @connection.dispatch_bytes(cmd.serialize.to_s)
+              message.documents.first
+            rescue Mongo::Error
+              @connection.disconnect!
+              @connection = nil
+              raise
             end
-            result = {}
+          end
+        else
+          connection = Connection.new(server.address, options)
+          connection.connect!
+          result = server.round_trip_time_averager.measure do
+            connection.handshake!
+          end
+          @connection = connection
+          if tv_doc = result['topologyVersion']
+            # Successful response, server 4.4+
+            create_push_monitor!(TopologyVersion.new(tv_doc))
+            push_monitor.run!
           else
-            if monitoring.monitoring?
-              monitoring.succeeded(
-                Monitoring::SERVER_HEARTBEAT,
-                Monitoring::Event::ServerHeartbeatSucceeded.new(server.address, rtt)
-              )
-            end
+            # Failed response or pre-4.4 server
+            stop_push_monitor!
           end
           result
         end
+        result
       end
 
       # @note If the system clock is set to a time in the past, this method
       #   can sleep for a very long time.
       def throttle_scan_frequency!
-        # Normally server.last_scan indicates when the previous scan
-        # completed, but if scan! is manually invoked repeatedly then
-        # server.last_scan won't be updated and multiple scans with no
-        # cooldown can be obtained. Guard against repeated direct scan!
-        # invocation also.
-        last_time = [server.last_scan, @scan_started_at].compact.max
-        if last_time
-          difference = (Time.now - last_time)
-          throttle_time = (MIN_SCAN_FREQUENCY - difference)
-          sleep(throttle_time) if throttle_time > 0
+        delta = @next_earliest_scan - Time.now
+        if delta > 0
+          sleep(delta)
         end
-        @scan_started_at = Time.now
       end
     end
   end

data/lib/mongo/server/pending_connection.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2019-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -34,9 +37,252 @@ module Mongo
       #   PendingConnection instance was created.
       attr_reader :id
 
+      def handshake_and_authenticate!
+        speculative_auth_doc = nil
+        if options[:user] || options[:auth_mech]
+          # To create an Auth instance, we need to specify the mechanism,
+          # but at this point we don't know the mechanism that ultimately
+          # will be used (since this depends on the data returned by
+          # the handshake, specifically server version).
+          # However, we know that only 4.4+ servers support speculative
+          # authentication, and those servers also generally support
+          # SCRAM-SHA-256. We expect that user accounts created for 4.4+
+          # servers would generally allow SCRAM-SHA-256 authentication;
+          # user accounts migrated from pre-4.4 servers may only allow
+          # SCRAM-SHA-1. The use of SCRAM-SHA-256 by default is thus
+          # sensible, and it is also mandated by the speculative auth spec.
+          # If no mechanism was specified and we are talking to a 3.0+
+          # server, we'll send speculative auth document, the server will
+          # ignore it and we'll perform authentication using explicit
+          # command after having defaulted the mechanism later to CR.
+          # If no mechanism was specified and we are talking to a 4.4+
+          # server and the user account doesn't allow SCRAM-SHA-256, we will
+          # authenticate in a separate command with SCRAM-SHA-1 after
+          # going through SCRAM mechanism negotiation.
+          default_options = Options::Redacted.new(:auth_mech => :scram256)
+          speculative_auth_user = Auth::User.new(default_options.merge(options))
+          speculative_auth = Auth.get(speculative_auth_user, self)
+          speculative_auth_doc = speculative_auth.conversation.speculative_auth_document
+        end
+
+        result = handshake!(speculative_auth_doc: speculative_auth_doc)
+
+        if description.unknown?
+          raise Error::InternalDriverError, "Connection description cannot be unknown after successful handshake: #{description.inspect}"
+        end
+
+        begin
+          if speculative_auth_doc && (speculative_auth_result = result['speculativeAuthenticate'])
+            unless description.features.scram_sha_1_enabled?
+              raise Error::InvalidServerAuthResponse, "Speculative auth succeeded on a pre-3.0 server"
+            end
+            case speculative_auth_user.mechanism
+            when :mongodb_x509
+              # Done
+            # We default auth mechanism to scram256, but if user specified
+            # scram explicitly we may be able to authenticate speculatively
+            # with scram.
+            when :scram, :scram256
+              authenticate!(
+                speculative_auth_client_nonce: speculative_auth.conversation.client_nonce,
+                speculative_auth_mech: speculative_auth_user.mechanism,
+                speculative_auth_result: speculative_auth_result,
+              )
+            else
+              raise Error::InternalDriverError, "Speculative auth unexpectedly succeeded for mechanism #{speculative_auth_user.mechanism.inspect}"
+            end
+          elsif !description.arbiter?
+            authenticate!
+          end
+        rescue Mongo::Error, Mongo::Error::AuthError => exc
+          exc.service_id = service_id
+          raise
+        end
+
+        if description.unknown?
+          raise Error::InternalDriverError, "Connection description cannot be unknown after successful authentication: #{description.inspect}"
+        end
+
+        if server.load_balancer? && !description.mongos?
+          raise Error::BadLoadBalancerTarget, "Load-balanced operation requires being connected a mongos, but the server at #{address.seed} reported itself as #{description.server_type.to_s.gsub('_', ' ')}"
+        end
+      end
+
+      private
+
+      # @param [ BSON::Document | nil ] speculative_auth_doc The document to
+      #   provide in speculativeAuthenticate field of handshake command.
+      #
+      # @return [ BSON::Document ] The document of the handshake response for
+      #   this particular connection.
+      def handshake!(speculative_auth_doc: nil)
+        unless socket
+          raise Error::InternalDriverError, "Cannot handshake because there is no usable socket (for #{address})"
+        end
+
+        hello_command = handshake_command(
+          handshake_document(
+            app_metadata,
+            speculative_auth_doc: speculative_auth_doc,
+            load_balancer: server.load_balancer?,
+            server_api: options[:server_api]
+          )
+        )
+        doc = nil
+        @server.handle_handshake_failure! do
+          begin
+            response = @server.round_trip_time_averager.measure do
+              add_server_diagnostics do
+                socket.write(hello_command.serialize.to_s)
+                Protocol::Message.deserialize(socket, Protocol::Message::MAX_MESSAGE_SIZE)
+              end
+            end
+            result = Operation::Result.new([response])
+            result.validate!
+            doc = result.documents.first
+          rescue => exc
+            msg = "Failed to handshake with #{address}"
+            Utils.warn_bg_exception(msg, exc,
+              logger: options[:logger],
+              log_prefix: options[:log_prefix],
+              bg_error_backtrace: options[:bg_error_backtrace],
+            )
+            raise
+          end
+        end
+
+        if @server.force_load_balancer?
+          doc['serviceId'] ||= "fake:#{rand(2**32-1)+1}"
+        end
+
+        post_handshake(doc, @server.round_trip_time_averager.average_round_trip_time)
+
+        doc
+      end
+
+      # @param [ String | nil ] speculative_auth_client_nonce The client
+      #   nonce used in speculative auth on this connection that
+      #   produced the specified speculative auth result.
+      # @param [ Symbol | nil ] speculative_auth_mech Auth mechanism used
+      #   for speculative auth, if speculative auth succeeded. If speculative
+      #   auth was not performed or it failed, this must be nil.
+      # @param [ BSON::Document | nil ] speculative_auth_result The
+      #   value of speculativeAuthenticate field of hello response of
+      #   the handshake on this connection.
+      def authenticate!(
+        speculative_auth_client_nonce: nil,
+        speculative_auth_mech: nil,
+        speculative_auth_result: nil
+      )
+        if options[:user] || options[:auth_mech]
+          @server.handle_auth_failure! do
+            begin
+              auth = Auth.get(
+                resolved_user(speculative_auth_mech: speculative_auth_mech),
+                self,
+                speculative_auth_client_nonce: speculative_auth_client_nonce,
+                speculative_auth_result: speculative_auth_result,
+              )
+              auth.login
+            rescue => exc
+              msg = "Failed to authenticate to #{address}"
+              Utils.warn_bg_exception(msg, exc,
+                logger: options[:logger],
+                log_prefix: options[:log_prefix],
+                bg_error_backtrace: options[:bg_error_backtrace],
+              )
+              raise
+            end
+          end
+        end
+      end
+
       def ensure_connected
         yield @socket
       end
+
+      # This is a separate method to keep the nesting level down.
+      #
+      # @return [ Server::Description ] The server description calculated from
+      #   the handshake response for this particular connection.
+      def post_handshake(response, average_rtt)
+        if response["ok"] == 1
+          # Auth mechanism is entirely dependent on the contents of
+          # hello response *for this connection*.
+          # Hello received by the monitoring connection should advertise
+          # the same wire protocol, but if it doesn't, we use whatever
+          # the monitoring connection advertised for filling out the
+          # server description and whatever the non-monitoring connection
+          # (that's this one) advertised for performing auth on that
+          # connection.
+          @sasl_supported_mechanisms = response['saslSupportedMechs']
+          set_compressor!(response)
+        else
+          @sasl_supported_mechanisms = nil
+        end
+
+        @description = Description.new(
+          address, response,
+          average_round_trip_time: average_rtt,
+          load_balancer: server.load_balancer?,
+          force_load_balancer: options[:connect] == :load_balanced,
+        ).tap do |new_description|
+          @server.cluster.run_sdam_flow(@server.description, new_description)
+        end
+      end
+
+      # The user as going to be used for authentication. This user has the
+      # auth mechanism set and, if necessary, auth source.
+      #
+      # @param [ Symbol | nil ] speculative_auth_mech Auth mechanism used
+      #   for speculative auth, if speculative auth succeeded. If speculative
+      #   auth was not performed or it failed, this must be nil.
+      #
+      # @return [ Auth::User ] The resolved user.
+      def resolved_user(speculative_auth_mech: nil)
+        @resolved_user ||= begin
+          unless options[:user] || options[:auth_mech]
+            raise Mongo::Error, 'No authentication information specified in the client'
+          end
+
+          user_options = Options::Redacted.new(
+            # When speculative auth is performed, we always use SCRAM-SHA-256.
+            # At the same time we perform SCRAM mechanism negotiation in the
+            # hello request.
+            # If the credentials we are trying to authenticate with do not
+            # map to an existing user, SCRAM mechanism negotiation will not
+            # return anything which would cause the driver to use
+            # SCRAM-SHA-1. However, on 4.4+ servers speculative auth would
+            # succeed (technically just the first round-trip, not the entire
+            # authentication flow) and we would be continuing it here;
+            # in this case, we must use SCRAM-SHA-256 as the mechanism since
+            # that is what the conversation was started with, even though
+            # SCRAM mechanism negotiation did not return SCRAM-SHA-256 as a
+            # valid mechanism to use for these credentials.
+            :auth_mech => speculative_auth_mech || default_mechanism,
+          ).merge(options)
+          if user_options[:auth_mech] == :mongodb_x509
+            user_options[:auth_source] = '$external'
+          end
+          Auth::User.new(user_options)
+        end
+      end
+
+      def default_mechanism
+        if description.nil?
+          raise Mongo::Error, 'Trying to query default mechanism when handshake has not completed'
+        end
+
+        if description.features.scram_sha_1_enabled?
+          if @sasl_supported_mechanisms&.include?('SCRAM-SHA-256')
+            :scram256
+          else
+            :scram
+          end
+        else
+          :mongodb_cr
+        end
+      end
     end
   end
 end

data/lib/mongo/server/push_monitor/connection.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Server
+    class PushMonitor
+
+      # @api private
+      class Connection < Server::Monitor::Connection
+
+        def socket_timeout
+          options[:socket_timeout]
+        end
+      end
+    end
+  end
+end