mongo 2.12.1 → 2.13.0

data/lib/mongo/server/connection_base.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2019 MongoDB, Inc.
+# Copyright (C) 2019-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -57,6 +57,7 @@ module Mongo
       # @return [ Mongo::Address ] address The address to connect to.
       def_delegators :server, :address
 
+      # @deprecated
       def_delegators :server,
                      :cluster_time,
                      :update_cluster_time
@@ -77,12 +78,21 @@ module Mongo
       # @api private
       attr_reader :description
 
+      # @deprecated
       def_delegators :description,
         :features,
         :max_bson_object_size,
         :max_message_size,
         :mongos?
 
+      # Connection pool generation from which this connection was created.
+      # May be nil.
+      #
+      # @return [ Integer | nil ] Connection pool generation.
+      def generation
+        options[:generation]
+      end
+
       def app_metadata
         @app_metadata ||= begin
           same = true
@@ -145,15 +155,20 @@ module Mongo
         ensure_connected do |socket|
           operation_id = Monitoring.next_operation_id
           command_started(address, operation_id, message.payload,
-            socket_object_id: socket.object_id, connection_id: id)
+            socket_object_id: socket.object_id, connection_id: id,
+            connection_generation: generation,
+            server_connection_id: description.server_connection_id,
+          )
           start = Time.now
           result = nil
           begin
-            socket.write(buffer.to_s)
-            result = if message.replyable?
-              Protocol::Message.deserialize(socket, max_message_size, message.request_id, options)
-            else
-              nil
+            result = add_server_diagnostics do
+              socket.write(buffer.to_s)
+              if message.replyable?
+                Protocol::Message.deserialize(socket, max_message_size, message.request_id, options)
+              else
+                nil
+              end
             end
           rescue Exception => e
             total_duration = Time.now - start
@@ -171,9 +186,6 @@ module Mongo
       end
 
       def serialize(message, client, buffer = BSON::ByteBuffer.new)
-        start_size = 0
-        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
-
         # Driver specifications only mandate the fixed 16MiB limit for
         # serialized BSON documents. However, the server returns its
         # active serialized BSON document size limit in the ismaster response,
@@ -198,12 +210,41 @@ module Mongo
           max_bson_size += MAX_BSON_COMMAND_OVERHEAD
         end
 
-        final_message.serialize(buffer, max_bson_size)
-        if max_message_size &&
-          (buffer.length - start_size) > max_message_size
-        then
-          raise Error::MaxMessageSize.new(max_message_size)
+        # RUBY-2234: It is necessary to check that the message size does not
+        # exceed the maximum bson object size before compressing and serializing
+        # the final message.
+        #
+        # This is to avoid the case where the user performs a bulk write
+        # larger than 16MiB which, when compressed, becomes smaller than 16MiB.
+        # If the driver does not split the bulk writes prior to compression,
+        # the entire operation will be sent to the server, which will raise an
+        # error because the uncompressed operation exceeds the maximum bson size.
+        #
+        # To address this problem, we serialize the message prior to compression
+        # and raise an exception if the serialized message exceeds the maximum
+        # bson size.
+        if max_message_size
+          # Create a separate buffer that contains the un-compressed message
+          # for the purpose of checking its size. Write any pre-existing contents
+          # from the original buffer into the temporary one.
+          temp_buffer = BSON::ByteBuffer.new
+
+          # TODO: address the fact that this line mutates the buffer.
+          temp_buffer.put_bytes(buffer.get_bytes(buffer.length))
+
+          message.serialize(temp_buffer, max_bson_size)
+          if temp_buffer.length > max_message_size
+            raise Error::MaxMessageSize.new(max_message_size)
+          end
         end
+
+        # RUBY-2335: When the un-compressed message is smaller than the maximum
+        # bson size limit, the message will be serialized twice. The operations
+        # layer should be refactored to allow compression on an already-
+        # serialized message.
+        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
+        final_message.serialize(buffer, max_bson_size)
+
         buffer
       end
     end

data/lib/mongo/server/connection_common.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 MongoDB, Inc.
+# Copyright (C) 2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,8 +33,26 @@ module Mongo
       # @return [ String | nil ] The compressor.
       attr_reader :compressor
 
+      # Determine if the connection is currently connected.
+      #
+      # @example Is the connection connected?
+      #   connection.connected?
+      #
+      # @return [ true, false ] If connected.
+      #
+      # @deprecated
+      def connected?
+        !!socket
+      end
+
+      # @return [ Integer ] pid The process id when the connection was created.
+      # @api private
+      attr_reader :pid
+
       private
 
+      attr_reader :socket
+
       def set_compressor!(reply)
         server_compressors = reply['compression']
 
@@ -56,6 +74,62 @@ module Mongo
           end
         end
       end
+
+      # Yields to the block and, if the block raises an exception, adds a note
+      # to the exception with the address of the specified server.
+      #
+      # This method is intended to add server address information to exceptions
+      # raised during execution of operations on servers.
+      def add_server_diagnostics
+        yield
+      # Note that the exception should already have been mapped to a
+      # Mongo::Error subclass when it gets to this method.
+      rescue Error::SocketError, Error::SocketTimeoutError => e
+        # Server::Monitor::Connection does not reference its server, but
+        # knows its address. Server::Connection delegates the address to its
+        # server.
+        note = "on #{address.seed}"
+        if respond_to?(:id)
+          note << ", connection #{generation}:#{id}"
+        end
+        e.add_note(note)
+        if respond_to?(:generation)
+          # Non-monitoring connections
+          e.generation = generation
+        end
+        raise e
+      end
+
+      def ssl_options
+        @ssl_options ||= if options[:ssl]
+          options.select { |k, v| k.to_s.start_with?('ssl') }
+        else
+          # Due to the way options are propagated from the client, if we
+          # decide that we don't want to use TLS we need to have the ssl
+          # options explicitly set to false or the value provided to the
+          # connection might be overwritten by the default inherited from
+          # the client.
+          {ssl: false}
+        end.freeze
+      end
+
+      def ensure_connected
+        begin
+          unless socket
+            raise ArgumentError, "Connection #{generation}:#{id} for #{address.seed} is not connected"
+          end
+          if @error
+            raise Error::ConnectionPerished, "Connection #{generation}:#{id} for #{address.seed} is perished"
+          end
+          result = yield socket
+          success = true
+          result
+        ensure
+          unless success
+            @error = true
+          end
+        end
+      end
     end
   end
 end

data/lib/mongo/server/connection_pool.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -292,6 +292,7 @@ module Mongo
         end
 
         deadline = Time.now + wait_timeout
+        pid = Process.pid
         connection = nil
         # It seems that synchronize sets up its own loop, thus a simple break
         # is insufficient to break the outer loop
@@ -304,6 +305,13 @@ module Mongo
               until @available_connections.empty?
                 connection = @available_connections.pop
 
+                if connection.pid != pid
+                  log_warn("Detected PID change - Mongo client should have been reconnected (old pid #{connection.pid}, new pid #{pid}")
+                  connection.disconnect!(reason: :stale)
+                  @populate_semaphore.signal
+                  next
+                end
+
                 if connection.generation != generation
                   # Stale connections should be disconnected in the clear
                   # method, but if any don't, check again here
@@ -383,6 +391,12 @@ module Mongo
           Monitoring::Event::Cmap::ConnectionCheckedOut.new(@server.address, connection.id, self),
         )
 
+        if Lint.enabled?
+          unless connection.connected?
+            raise Error::LintError, "Connection pool for #{address} checked out a disconnected connection #{connection.generation}:#{connection.id}"
+          end
+        end
+
         connection
       ensure
         check_invariants
@@ -418,6 +432,11 @@ module Mongo
             Monitoring::Event::Cmap::ConnectionCheckedIn.new(@server.address, connection.id, self)
           )
 
+          if connection.error?
+            connection.disconnect!(reason: :error)
+            return
+          end
+
           if closed?
             connection.disconnect!(reason: :pool_closed)
             return

data/lib/mongo/server/connection_pool/populator.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2019 MongoDB, Inc.
+# Copyright (C) 2019-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/server/context.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.

data/lib/mongo/server/description.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.
@@ -214,6 +214,7 @@ module Mongo
           hosts
           arbiters
           passives
+          topology_version
 
           freeze
         end
@@ -419,6 +420,47 @@ module Mongo
         config[SET_VERSION]
       end
 
+      # @return [ TopologyVersion | nil ] The topology version.
+      def topology_version
+        unless defined?(@topology_version)
+          @topology_version = config['topologyVersion'] &&
+            TopologyVersion.new(config['topologyVersion'])
+        end
+        @topology_version
+      end
+
+      # Returns whether topology version in this description is potentially
+      # newer than or equal to topology version in another description.
+      #
+      # @param [ Server::Description ] other_desc The other server description.
+      #
+      # @return [ true | false ] Whether topology version in this description
+      #   is potentially newer or equal.
+      # @api private
+      def topology_version_gt?(other_desc)
+        if topology_version.nil? || other_desc.topology_version.nil?
+          true
+        else
+          topology_version.gt?(other_desc.topology_version)
+        end
+      end
+
+      # Returns whether topology version in this description is potentially
+      # newer than topology version in another description.
+      #
+      # @param [ Server::Description ] other_desc The other server description.
+      #
+      # @return [ true | false ] Whether topology version in this description
+      #   is potentially newer.
+      # @api private
+      def topology_version_gte?(other_desc)
+        if topology_version.nil? || other_desc.topology_version.nil?
+          true
+        else
+          topology_version.gte?(other_desc.topology_version)
+        end
+      end
+
       # Get the lastWriteDate from the lastWrite subdocument in the config.
       #
       # @example Get the lastWriteDate value.
@@ -606,7 +648,7 @@ module Mongo
       #
       # @since 2.0.0
       def unknown?
-        config.empty? || !ok?
+        config.empty? || config.keys == %w(topologyVersion) || !ok?
       end
 
       # @api private
@@ -713,6 +755,11 @@ module Mongo
       # @since 2.7.0
       attr_reader :last_update_time
 
+      # @api experimental
+      def server_connection_id
+        config['connectionId']
+      end
+
       # Check equality of two descriptions.
       #
       # @example Check description equality.

data/lib/mongo/server/description/features.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.
@@ -20,7 +20,6 @@ module Mongo
       #
       # @since 2.0.0
       class Features
-
         # List of features and the wire protocol version they appear in.
         #
         # Wire protocol versions map to server releases as follows:
@@ -31,9 +30,15 @@ module Mongo
         # - 6 => 3.6
         # - 7 => 4.0
         # - 8 => 4.2
+        # - 9 => 4.4
         #
         # @since 2.0.0
         MAPPINGS = {
+          :retryable_write_error_label => 9,
+          # Server versions older than 4.2 do not reliably validate options
+          # provided by the client during findAndModify operations, requiring the
+          # driver to raise client-side errors when those options are provided.
+          :find_and_modify_option_validation => 8,
           :transactions => 7,
           :scram_sha_256 => 7,
           :array_filters => 6,
@@ -41,6 +46,10 @@ module Mongo
           :sessions => 6,
           :collation => 5,
           :max_staleness => 5,
+          # Server versions older than 3.4 do not reliably validate options
+          # provided by the client during update/delete operations, requiring the
+          # driver to raise client-side errors when those options are provided.
+          :update_delete_option_validation => 5,
           :find_command => 4,
           :list_collections => 3,
           :list_indexes => 3,
@@ -64,7 +73,7 @@ module Mongo
         # The wire protocol versions that this version of the driver supports.
         #
         # @since 2.0.0
-        DRIVER_WIRE_VERSIONS = (2..8).freeze
+        DRIVER_WIRE_VERSIONS = (2..9).freeze
 
         # Create the methods for each mapping to tell if they are supported.
         #

data/lib/mongo/server/monitor.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,22 +22,23 @@ module Mongo
     # operations performed by the driver.
     #
     # @since 2.0.0
+    # @api private
     class Monitor
       include Loggable
       extend Forwardable
       include Event::Publisher
       include BackgroundThread
 
-      # The default time for a server to refresh its status is 10 seconds.
+      # The default interval between server status refreshes is 10 seconds.
       #
       # @since 2.0.0
-      HEARTBEAT_FREQUENCY = 10.freeze
+      DEFAULT_HEARTBEAT_INTERVAL = 10.freeze
 
       # The minimum time between forced server scans. Is
       # minHeartbeatFrequencyMS in the SDAM spec.
       #
       # @since 2.0.0
-      MIN_SCAN_FREQUENCY = 0.5.freeze
+      MIN_SCAN_INTERVAL = 0.5.freeze
 
       # The weighting factor (alpha) for calculating the average moving round trip time.
       #
@@ -59,6 +60,8 @@ module Mongo
       #
       # @option options [ Float ] :connect_timeout The timeout, in seconds, to
       #   use when establishing the monitoring connection.
+      # @option options [ Float ] :heartbeat_interval The interval between
+      #   regular server checks.
       # @option options [ Logger ] :logger A custom logger to use.
       # @option options [ Float ] :socket_timeout The timeout, in seconds, to
       #   execute operations on the monitoring connection.
@@ -73,10 +76,10 @@ module Mongo
         @event_listeners = event_listeners
         @monitoring = monitoring
         @options = options.freeze
-        # This is a Mongo::Server::Monitor::Connection
-        @connection = Connection.new(server.address, options)
         @mutex = Mutex.new
-        @scan_started_at = nil
+        @sdam_mutex = Mutex.new
+        @next_earliest_scan = @next_wanted_scan = Time.now
+        @update_mutex = Mutex.new
       end
 
       # @return [ Server ] server The server that this monitor is monitoring.
@@ -89,11 +92,18 @@ module Mongo
       # @return [ Hash ] options The server options.
       attr_reader :options
 
+      # The interval between regular server checks.
+      #
+      # @return [ Float ] The heartbeat interval, in seconds.
+      def heartbeat_interval
+        options[:heartbeat_interval] || DEFAULT_HEARTBEAT_INTERVAL
+      end
+
       # @deprecated
       def_delegators :server, :last_scan
 
-      # The compressor is determined during the handshake, so it must be an attribute
-      # of the connection.
+      # The compressor is determined during the handshake, so it must be an
+      # attribute of the connection.
       #
       # @deprecated
       def_delegators :connection, :compressor
@@ -101,15 +111,12 @@ module Mongo
       # @return [ Monitoring ] monitoring The monitoring.
       attr_reader :monitoring
 
-      # Get the refresh interval for the server. This will be defined via an
-      # option or will default to 10.
-      #
-      # @return [ Float ] The heartbeat interval, in seconds.
-      #
-      # @since 2.0.0
-      # @deprecated
-      def heartbeat_frequency
-        server.cluster.heartbeat_interval
+      # @return [ Server::PushMonitor | nil ] The push monitor, if one is being
+      #   used.
+      def push_monitor
+        @update_mutex.synchronize do
+          @push_monitor
+        end
       end
 
       # Runs the server monitor. Refreshing happens on a separate thread per
@@ -123,52 +130,99 @@ module Mongo
       # @since 2.0.0
       def do_work
         scan!
-        server.scan_semaphore.wait(server.cluster.heartbeat_interval)
+        # @next_wanted_scan may be updated by the push monitor.
+        # However we need to check for termination flag so that the monitor
+        # thread exits when requested.
+        loop do
+          delta = @next_wanted_scan - Time.now
+          if delta > 0
+            signaled = server.scan_semaphore.wait(delta)
+            if signaled || @stop_requested
+              break
+            end
+          else
+            break
+          end
+        end
       end
 
-      # Stop the background thread and wait for to terminate for a reasonable
-      # amount of time.
+      # Stop the background thread and wait for it to terminate for a
+      # reasonable amount of time.
       #
       # @return [ true | false ] Whether the thread was terminated.
       #
       # @api public for backwards compatibility only
       def stop!
+        stop_push_monitor!
+
         # Forward super's return value
         super.tap do
           # Important: disconnect should happen after the background thread
-          # terminated.
-          connection.disconnect!
+          # terminates.
+          connection&.disconnect!
+        end
+      end
+
+      def stop_push_monitor!
+        @update_mutex.synchronize do
+          if @push_monitor
+            @push_monitor.stop!
+            @push_monitor = nil
+          end
         end
       end
 
       # Perform a check of the server with throttling, and update
       # the server's description and average round trip time.
       #
-      # If the server was checked less than MIN_SCAN_FREQUENCY seconds
-      # ago, sleep until MIN_SCAN_FREQUENCY seconds have passed since the last
+      # If the server was checked less than MIN_SCAN_INTERVAL seconds
+      # ago, sleep until MIN_SCAN_INTERVAL seconds have passed since the last
       # check. Then perform the check which involves running isMaster
       # on the server being monitored and updating the server description
       # as a result.
       #
-      # @note If the system clock is set to a time in the past, this method
-      #   can sleep for a very long time.
+      # @note If the system clock moves backwards, this method can sleep
+      #   for a very long time.
       #
       # @note The return value of this method is deprecated. In version 3.0.0
       #   this method will not have a return value.
       #
-      # @example Run a scan.
-      #   monitor.scan!
-      #
       # @return [ Description ] The updated description.
       #
       # @since 2.0.0
       def scan!
-        throttle_scan_frequency!
-        result = ismaster
-        new_description = Description.new(server.address, result,
-          server.round_trip_time_averager.average_round_trip_time)
-        server.cluster.run_sdam_flow(server.description, new_description)
-        server.description
+        # Ordinarily the background thread would invoke this method.
+        # But it is also possible to invoke scan! directly on a monitor.
+        # Allow only one scan to be performed at a time.
+        @mutex.synchronize do
+          throttle_scan_frequency!
+
+          result = do_scan
+
+          run_sdam_flow(result)
+        end
+      end
+
+      def run_sdam_flow(result, awaited: false)
+        @sdam_mutex.synchronize do
+          old_description = server.description
+
+          new_description = Description.new(server.address, result,
+            server.round_trip_time_averager.average_round_trip_time)
+
+          server.cluster.run_sdam_flow(server.description, new_description, awaited: awaited)
+
+          server.description.tap do |new_description|
+            unless awaited
+              if new_description.unknown? && !old_description.unknown?
+                @next_earliest_scan = @next_wanted_scan = Time.now
+              else
+                @next_earliest_scan = Time.now + MIN_SCAN_INTERVAL
+                @next_wanted_scan = Time.now + heartbeat_interval
+              end
+            end
+          end
+        end
       end
 
       # Restarts the server monitor unless the current thread is alive.
@@ -193,54 +247,76 @@ module Mongo
         server.scan_semaphore.signal
       end
 
-      def ismaster
-        @mutex.synchronize do
-          if monitoring.monitoring?
-            monitoring.started(
-              Monitoring::SERVER_HEARTBEAT,
-              Monitoring::Event::ServerHeartbeatStarted.new(server.address)
-            )
+      def do_scan
+        begin
+          monitoring.publish_heartbeat(server) do
+            ismaster
           end
+        rescue => exc
+          msg = "Error running ismaster on #{server.address}"
+          Utils.warn_monitor_exception(msg, exc,
+            logger: options[:logger],
+            log_prefix: options[:log_prefix],
+            bg_error_backtrace: options[:bg_error_backtrace],
+          )
+          {}
+        end
+      end
+
+      def ismaster
+        if @connection && @connection.pid != Process.pid
+          log_warn("Detected PID change - Mongo client should have been reconnected (old pid #{@connection.pid}, new pid #{Process.pid}")
+          @connection.disconnect!
+          @connection = nil
+        end
 
-          result, exc, rtt, average_rtt = server.round_trip_time_averager.measure do
-            connection.ismaster
+        if @connection
+          result = server.round_trip_time_averager.measure do
+            begin
+              message = @connection.dispatch_bytes(Monitor::Connection::ISMASTER_BYTES)
+              message.documents.first
+            rescue Mongo::Error
+              @connection.disconnect!
+              @connection = nil
+              raise
+            end
+          end
+        else
+          connection = Connection.new(server.address, options)
+          connection.connect!
+          result = server.round_trip_time_averager.measure do
+            connection.handshake!
           end
-          if exc
-            log_debug("Error running ismaster on #{server.address}: #{exc.class}: #{exc}:\n#{exc.backtrace[0..5].join("\n")}")
-            if monitoring.monitoring?
-              monitoring.failed(
-                Monitoring::SERVER_HEARTBEAT,
-                Monitoring::Event::ServerHeartbeatFailed.new(server.address, rtt, exc)
+          @connection = connection
+          if result['topologyVersion']
+            # Successful response, server 4.4+
+            @update_mutex.synchronize do
+              @push_monitor ||= PushMonitor.new(
+                self,
+                TopologyVersion.new(result['topologyVersion']),
+                monitoring,
+                **Utils.shallow_symbolize_keys(options.merge(
+                  socket_timeout: heartbeat_interval + connection.socket_timeout,
+                )),
               )
             end
-            result = {}
+            push_monitor.run!
           else
-            if monitoring.monitoring?
-              monitoring.succeeded(
-                Monitoring::SERVER_HEARTBEAT,
-                Monitoring::Event::ServerHeartbeatSucceeded.new(server.address, rtt)
-              )
-            end
+            # Failed response or pre-4.4 server
+            stop_push_monitor!
           end
           result
         end
+        result
       end
 
       # @note If the system clock is set to a time in the past, this method
       #   can sleep for a very long time.
       def throttle_scan_frequency!
-        # Normally server.last_scan indicates when the previous scan
-        # completed, but if scan! is manually invoked repeatedly then
-        # server.last_scan won't be updated and multiple scans with no
-        # cooldown can be obtained. Guard against repeated direct scan!
-        # invocation also.
-        last_time = [server.last_scan, @scan_started_at].compact.max
-        if last_time
-          difference = (Time.now - last_time)
-          throttle_time = (MIN_SCAN_FREQUENCY - difference)
-          sleep(throttle_time) if throttle_time > 0
+        delta = @next_earliest_scan - Time.now
+        if delta > 0
+          sleep(delta)
         end
-        @scan_started_at = Time.now
       end
     end
   end