mongo 2.12.4 → 2.13.0.beta1

data/lib/mongo/server/connection_base.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2019 MongoDB, Inc.
+# Copyright (C) 2019-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -57,6 +57,7 @@ module Mongo
       # @return [ Mongo::Address ] address The address to connect to.
       def_delegators :server, :address
 
+      # @deprecated
       def_delegators :server,
                      :cluster_time,
                      :update_cluster_time
@@ -77,12 +78,21 @@ module Mongo
       # @api private
       attr_reader :description
 
+      # @deprecated
       def_delegators :description,
         :features,
         :max_bson_object_size,
         :max_message_size,
         :mongos?
 
+      # Connection pool generation from which this connection was created.
+      # May be nil.
+      #
+      # @return [ Integer | nil ] Connection pool generation.
+      def generation
+        options[:generation]
+      end
+
       def app_metadata
         @app_metadata ||= begin
           same = true
@@ -145,15 +155,18 @@ module Mongo
         ensure_connected do |socket|
           operation_id = Monitoring.next_operation_id
           command_started(address, operation_id, message.payload,
-            socket_object_id: socket.object_id, connection_id: id)
+            socket_object_id: socket.object_id, connection_id: id,
+            connection_generation: generation)
           start = Time.now
           result = nil
           begin
-            socket.write(buffer.to_s)
-            result = if message.replyable?
-              Protocol::Message.deserialize(socket, max_message_size, message.request_id, options)
-            else
-              nil
+            result = add_server_diagnostics do
+              socket.write(buffer.to_s)
+              if message.replyable?
+                Protocol::Message.deserialize(socket, max_message_size, message.request_id, options)
+              else
+                nil
+              end
             end
           rescue Exception => e
             total_duration = Time.now - start
@@ -171,6 +184,9 @@ module Mongo
       end
 
       def serialize(message, client, buffer = BSON::ByteBuffer.new)
+        start_size = 0
+        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
+
         # Driver specifications only mandate the fixed 16MiB limit for
         # serialized BSON documents. However, the server returns its
         # active serialized BSON document size limit in the ismaster response,
@@ -189,45 +205,18 @@ module Mongo
           if message.bulk_write?
             # Make the new maximum size equal to the specified reduced size
             # limit plus the 16KiB overhead allowance.
-            max_bson_size = REDUCED_MAX_BSON_SIZE
+            max_bson_size = REDUCED_MAX_BSON_SIZE + MAX_BSON_COMMAND_OVERHEAD
           end
+        else
+          max_bson_size += MAX_BSON_COMMAND_OVERHEAD
         end
 
-        # RUBY-2234: It is necessary to check that the message size does not
-        # exceed the maximum bson object size before compressing and serializing
-        # the final message.
-        #
-        # This is to avoid the case where the user performs a bulk write
-        # larger than 16MiB which, when compressed, becomes smaller than 16MiB.
-        # If the driver does not split the bulk writes prior to compression,
-        # the entire operation will be sent to the server, which will raise an
-        # error because the uncompressed operation exceeds the maximum bson size.
-        #
-        # To address this problem, we serialize the message prior to compression
-        # and raise an exception if the serialized message exceeds the maximum
-        # bson size.
-        if max_message_size
-          # Create a separate buffer that contains the un-compressed message
-          # for the purpose of checking its size. Write any pre-existing contents
-          # from the original buffer into the temporary one.
-          temp_buffer = BSON::ByteBuffer.new
-
-          # TODO: address the fact that this line mutates the buffer.
-          temp_buffer.put_bytes(buffer.get_bytes(buffer.length))
-
-          message.serialize(temp_buffer, max_bson_size, MAX_BSON_COMMAND_OVERHEAD)
-          if temp_buffer.length > max_message_size
-            raise Error::MaxMessageSize.new(max_message_size)
-          end
+        final_message.serialize(buffer, max_bson_size)
+        if max_message_size &&
+          (buffer.length - start_size) > max_message_size
+        then
+          raise Error::MaxMessageSize.new(max_message_size)
         end
-
-        # RUBY-2335: When the un-compressed message is smaller than the maximum
-        # bson size limit, the message will be serialized twice. The operations
-        # layer should be refactored to allow compression on an already-
-        # serialized message.
-        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
-        final_message.serialize(buffer, max_bson_size, MAX_BSON_COMMAND_OVERHEAD)
-
         buffer
       end
     end

data/lib/mongo/server/connection_common.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2020 MongoDB, Inc.
+# Copyright (C) 2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -33,8 +33,26 @@ module Mongo
       # @return [ String | nil ] The compressor.
       attr_reader :compressor
 
+      # Determine if the connection is currently connected.
+      #
+      # @example Is the connection connected?
+      #   connection.connected?
+      #
+      # @return [ true, false ] If connected.
+      #
+      # @deprecated
+      def connected?
+        !!socket
+      end
+
+      # @return [ Integer ] pid The process id when the connection was created.
+      # @api private
+      attr_reader :pid
+
       private
 
+      attr_reader :socket
+
       def set_compressor!(reply)
         server_compressors = reply['compression']
 
@@ -56,6 +74,62 @@ module Mongo
           end
         end
       end
+
+      # Yields to the block and, if the block raises an exception, adds a note
+      # to the exception with the address of the specified server.
+      #
+      # This method is intended to add server address information to exceptions
+      # raised during execution of operations on servers.
+      def add_server_diagnostics
+        yield
+      # Note that the exception should already have been mapped to a
+      # Mongo::Error subclass when it gets to this method.
+      rescue Error::SocketError, Error::SocketTimeoutError => e
+        # Server::Monitor::Connection does not reference its server, but
+        # knows its address. Server::Connection delegates the address to its
+        # server.
+        note = "on #{address.seed}"
+        if respond_to?(:id)
+          note << ", connection #{generation}:#{id}"
+        end
+        e.add_note(note)
+        if respond_to?(:generation)
+          # Non-monitoring connections
+          e.generation = generation
+        end
+        raise e
+      end
+
+      def ssl_options
+        @ssl_options ||= if options[:ssl]
+          options.select { |k, v| k.to_s.start_with?('ssl') }
+        else
+          # Due to the way options are propagated from the client, if we
+          # decide that we don't want to use TLS we need to have the ssl
+          # options explicitly set to false or the value provided to the
+          # connection might be overwritten by the default inherited from
+          # the client.
+          {ssl: false}
+        end.freeze
+      end
+
+      def ensure_connected
+        begin
+          unless socket
+            raise ArgumentError, "Connection #{generation}:#{id} for #{address.seed} is not connected"
+          end
+          if @error
+            raise Error::ConnectionPerished, "Connection #{generation}:#{id} for #{address.seed} is perished"
+          end
+          result = yield socket
+          success = true
+          result
+        ensure
+          unless success
+            @error = true
+          end
+        end
+      end
     end
   end
 end

data/lib/mongo/server/connection_pool.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -292,6 +292,7 @@ module Mongo
         end
 
         deadline = Time.now + wait_timeout
+        pid = Process.pid
         connection = nil
         # It seems that synchronize sets up its own loop, thus a simple break
         # is insufficient to break the outer loop
@@ -304,6 +305,13 @@ module Mongo
               until @available_connections.empty?
                 connection = @available_connections.pop
 
+                if connection.pid != pid
+                  log_warn("Detected PID change - Mongo client should have been reconnected (old pid #{connection.pid}, new pid #{pid}")
+                  connection.disconnect!(reason: :stale)
+                  @populate_semaphore.signal
+                  next
+                end
+
                 if connection.generation != generation
                   # Stale connections should be disconnected in the clear
                   # method, but if any don't, check again here
@@ -383,6 +391,12 @@ module Mongo
           Monitoring::Event::Cmap::ConnectionCheckedOut.new(@server.address, connection.id, self),
         )
 
+        if Lint.enabled?
+          unless connection.connected?
+            raise Error::LintError, "Connection pool for #{address} checked out a disconnected connection #{connection.generation}:#{connection.id}"
+          end
+        end
+
         connection
       ensure
         check_invariants
@@ -418,6 +432,11 @@ module Mongo
             Monitoring::Event::Cmap::ConnectionCheckedIn.new(@server.address, connection.id, self)
           )
 
+          if connection.error?
+            connection.disconnect!(reason: :error)
+            return
+          end
+
           if closed?
             connection.disconnect!(reason: :pool_closed)
             return

data/lib/mongo/server/connection_pool/populator.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2019 MongoDB, Inc.
+# Copyright (C) 2019-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/server/context.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.

data/lib/mongo/server/description.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.
@@ -214,6 +214,7 @@ module Mongo
           hosts
           arbiters
           passives
+          topology_version
 
           freeze
         end
@@ -419,6 +420,47 @@ module Mongo
         config[SET_VERSION]
       end
 
+      # @return [ TopologyVersion | nil ] The topology version.
+      def topology_version
+        unless defined?(@topology_version)
+          @topology_version = config['topologyVersion'] &&
+            TopologyVersion.new(config['topologyVersion'])
+        end
+        @topology_version
+      end
+
+      # Returns whether topology version in this description is potentially
+      # newer than or equal to topology version in another description.
+      #
+      # @param [ Server::Description ] other_desc The other server description.
+      #
+      # @return [ true | false ] Whether topology version in this description
+      #   is potentially newer or equal.
+      # @api private
+      def topology_version_gt?(other_desc)
+        if topology_version.nil? || other_desc.topology_version.nil?
+          true
+        else
+          topology_version.gt?(other_desc.topology_version)
+        end
+      end
+
+      # Returns whether topology version in this description is potentially
+      # newer than topology version in another description.
+      #
+      # @param [ Server::Description ] other_desc The other server description.
+      #
+      # @return [ true | false ] Whether topology version in this description
+      #   is potentially newer.
+      # @api private
+      def topology_version_gte?(other_desc)
+        if topology_version.nil? || other_desc.topology_version.nil?
+          true
+        else
+          topology_version.gte?(other_desc.topology_version)
+        end
+      end
+
       # Get the lastWriteDate from the lastWrite subdocument in the config.
       #
       # @example Get the lastWriteDate value.
@@ -606,7 +648,7 @@ module Mongo
       #
       # @since 2.0.0
       def unknown?
-        config.empty? || !ok?
+        config.empty? || config.keys == %w(topologyVersion) || !ok?
       end
 
       # @api private

data/lib/mongo/server/description/features.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.
@@ -20,7 +20,6 @@ module Mongo
       #
       # @since 2.0.0
       class Features
-
         # List of features and the wire protocol version they appear in.
         #
         # Wire protocol versions map to server releases as follows:
@@ -31,9 +30,15 @@ module Mongo
         # - 6 => 3.6
         # - 7 => 4.0
         # - 8 => 4.2
+        # - 9 => 4.4
         #
         # @since 2.0.0
         MAPPINGS = {
+          :retryable_write_error_label => 9,
+          # Server versions older than 4.2 do not reliably validate options
+          # provided by the client during findAndModify operations, requiring the
+          # driver to raise client-side errors when those options are provided.
+          :find_and_modify_option_validation => 8,
           :transactions => 7,
           :scram_sha_256 => 7,
           :array_filters => 6,
@@ -41,6 +46,10 @@ module Mongo
           :sessions => 6,
           :collation => 5,
           :max_staleness => 5,
+          # Server versions older than 3.4 do not reliably validate options
+          # provided by the client during update/delete operations, requiring the
+          # driver to raise client-side errors when those options are provided.
+          :update_delete_option_validation => 5,
           :find_command => 4,
           :list_collections => 3,
           :list_indexes => 3,
@@ -64,7 +73,7 @@ module Mongo
         # The wire protocol versions that this version of the driver supports.
         #
         # @since 2.0.0
-        DRIVER_WIRE_VERSIONS = (2..8).freeze
+        DRIVER_WIRE_VERSIONS = (2..9).freeze
 
         # Create the methods for each mapping to tell if they are supported.
         #

data/lib/mongo/server/monitor.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2019 MongoDB Inc.
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,22 +22,23 @@ module Mongo
     # operations performed by the driver.
     #
     # @since 2.0.0
+    # @api private
     class Monitor
       include Loggable
       extend Forwardable
       include Event::Publisher
       include BackgroundThread
 
-      # The default time for a server to refresh its status is 10 seconds.
+      # The default interval between server status refreshes is 10 seconds.
       #
       # @since 2.0.0
-      HEARTBEAT_FREQUENCY = 10.freeze
+      DEFAULT_HEARTBEAT_INTERVAL = 10.freeze
 
       # The minimum time between forced server scans. Is
       # minHeartbeatFrequencyMS in the SDAM spec.
       #
       # @since 2.0.0
-      MIN_SCAN_FREQUENCY = 0.5.freeze
+      MIN_SCAN_INTERVAL = 0.5.freeze
 
       # The weighting factor (alpha) for calculating the average moving round trip time.
       #
@@ -59,6 +60,8 @@ module Mongo
       #
       # @option options [ Float ] :connect_timeout The timeout, in seconds, to
       #   use when establishing the monitoring connection.
+      # @option options [ Float ] :heartbeat_interval The interval between
+      #   regular server checks.
       # @option options [ Logger ] :logger A custom logger to use.
       # @option options [ Float ] :socket_timeout The timeout, in seconds, to
       #   execute operations on the monitoring connection.
@@ -73,10 +76,8 @@ module Mongo
         @event_listeners = event_listeners
         @monitoring = monitoring
         @options = options.freeze
-        # This is a Mongo::Server::Monitor::Connection
-        @connection = Connection.new(server.address, options)
         @mutex = Mutex.new
-        @scan_started_at = nil
+        @next_earliest_scan = @next_wanted_scan = Time.now
       end
 
       # @return [ Server ] server The server that this monitor is monitoring.
@@ -89,11 +90,18 @@ module Mongo
       # @return [ Hash ] options The server options.
       attr_reader :options
 
+      # The interval between regular server checks.
+      #
+      # @return [ Float ] The heartbeat interval, in seconds.
+      def heartbeat_interval
+        options[:heartbeat_interval] || DEFAULT_HEARTBEAT_INTERVAL
+      end
+
       # @deprecated
       def_delegators :server, :last_scan
 
-      # The compressor is determined during the handshake, so it must be an attribute
-      # of the connection.
+      # The compressor is determined during the handshake, so it must be an
+      # attribute of the connection.
       #
       # @deprecated
       def_delegators :connection, :compressor
@@ -101,17 +109,6 @@ module Mongo
       # @return [ Monitoring ] monitoring The monitoring.
       attr_reader :monitoring
 
-      # Get the refresh interval for the server. This will be defined via an
-      # option or will default to 10.
-      #
-      # @return [ Float ] The heartbeat interval, in seconds.
-      #
-      # @since 2.0.0
-      # @deprecated
-      def heartbeat_frequency
-        server.cluster.heartbeat_interval
-      end
-
       # Runs the server monitor. Refreshing happens on a separate thread per
       # server.
       #
@@ -123,11 +120,14 @@ module Mongo
       # @since 2.0.0
       def do_work
         scan!
-        server.scan_semaphore.wait(server.cluster.heartbeat_interval)
+        delta = @next_wanted_scan - Time.now
+        if delta > 0
+          server.scan_semaphore.wait(delta)
+        end
       end
 
-      # Stop the background thread and wait for to terminate for a reasonable
-      # amount of time.
+      # Stop the background thread and wait for it to terminate for a
+      # reasonable amount of time.
       #
       # @return [ true | false ] Whether the thread was terminated.
       #
@@ -136,39 +136,54 @@ module Mongo
         # Forward super's return value
         super.tap do
           # Important: disconnect should happen after the background thread
-          # terminated.
-          connection.disconnect!
+          # terminates.
+          connection&.disconnect!
         end
       end
 
       # Perform a check of the server with throttling, and update
       # the server's description and average round trip time.
       #
-      # If the server was checked less than MIN_SCAN_FREQUENCY seconds
-      # ago, sleep until MIN_SCAN_FREQUENCY seconds have passed since the last
+      # If the server was checked less than MIN_SCAN_INTERVAL seconds
+      # ago, sleep until MIN_SCAN_INTERVAL seconds have passed since the last
       # check. Then perform the check which involves running isMaster
       # on the server being monitored and updating the server description
       # as a result.
       #
-      # @note If the system clock is set to a time in the past, this method
-      #   can sleep for a very long time.
+      # @note If the system clock moves backwards, this method can sleep
+      #   for a very long time.
       #
       # @note The return value of this method is deprecated. In version 3.0.0
       #   this method will not have a return value.
       #
-      # @example Run a scan.
-      #   monitor.scan!
-      #
       # @return [ Description ] The updated description.
       #
       # @since 2.0.0
       def scan!
-        throttle_scan_frequency!
-        result = ismaster
-        new_description = Description.new(server.address, result,
-          server.round_trip_time_averager.average_round_trip_time)
-        server.cluster.run_sdam_flow(server.description, new_description)
-        server.description
+        # Ordinarily the background thread would invoke this method.
+        # But it is also possible to invoke scan! directly on a monitor.
+        # Allow only one scan to be performed at a time.
+        @mutex.synchronize do
+          throttle_scan_frequency!
+
+          result = do_scan
+
+          old_description = server.description
+
+          new_description = Description.new(server.address, result,
+            server.round_trip_time_averager.average_round_trip_time)
+
+          server.cluster.run_sdam_flow(server.description, new_description)
+
+          server.description.tap do |new_description|
+            if new_description.unknown? && !old_description.unknown?
+              @next_earliest_scan = @next_wanted_scan = Time.now
+            else
+              @next_earliest_scan = Time.now + MIN_SCAN_INTERVAL
+              @next_wanted_scan = Time.now + heartbeat_interval
+            end
+          end
+        end
       end
 
       # Restarts the server monitor unless the current thread is alive.
@@ -193,54 +208,83 @@ module Mongo
         server.scan_semaphore.signal
       end
 
-      def ismaster
-        @mutex.synchronize do
+      def do_scan
+        if monitoring.monitoring?
+          monitoring.started(
+            Monitoring::SERVER_HEARTBEAT,
+            Monitoring::Event::ServerHeartbeatStarted.new(server.address)
+          )
+        end
+
+        # The duration we publish in heartbeat succeeded/failed events is
+        # the time spent on the entire heartbeat. This could include time
+        # to connect the socket (including TLS handshake), not just time
+        # spent on ismaster call itself.
+        # The spec at https://github.com/mongodb/specifications/blob/master/source/server-discovery-and-monitoring/server-discovery-and-monitoring-monitoring.rst
+        # requires that the duration exposed here start from "sending the
+        # message" (ismaster). This requirement does not make sense if,
+        # for example, we were never able to connect to the server at all
+        # and thus ismaster was never sent.
+        start_time = Time.now
+
+        begin
+          result = ismaster
+        rescue => exc
+          log_warn("Error running ismaster on #{server.address}: #{exc.class}: #{exc}:\n#{exc.backtrace[0..5].join("\n")}")
           if monitoring.monitoring?
-            monitoring.started(
+            monitoring.failed(
               Monitoring::SERVER_HEARTBEAT,
-              Monitoring::Event::ServerHeartbeatStarted.new(server.address)
+              Monitoring::Event::ServerHeartbeatFailed.new(server.address, Time.now-start_time, exc)
             )
           end
-
-          result, exc, rtt, average_rtt = server.round_trip_time_averager.measure do
-            connection.ismaster
+          result = {}
+        else
+          if monitoring.monitoring?
+            monitoring.succeeded(
+              Monitoring::SERVER_HEARTBEAT,
+              Monitoring::Event::ServerHeartbeatSucceeded.new(server.address, Time.now-start_time)
+            )
           end
-          if exc
-            log_debug("Error running ismaster on #{server.address}: #{exc.class}: #{exc}:\n#{exc.backtrace[0..5].join("\n")}")
-            if monitoring.monitoring?
-              monitoring.failed(
-                Monitoring::SERVER_HEARTBEAT,
-                Monitoring::Event::ServerHeartbeatFailed.new(server.address, rtt, exc)
-              )
-            end
-            result = {}
-          else
-            if monitoring.monitoring?
-              monitoring.succeeded(
-                Monitoring::SERVER_HEARTBEAT,
-                Monitoring::Event::ServerHeartbeatSucceeded.new(server.address, rtt)
-              )
+        end
+        result
+      end
+
+      def ismaster
+        if @connection && @connection.pid != Process.pid
+          log_warn("Detected PID change - Mongo client should have been reconnected (old pid #{@connection.pid}, new pid #{Process.pid}")
+          @connection.disconnect!
+          @connection = nil
+        end
+
+        if @connection
+          result = server.round_trip_time_averager.measure do
+            begin
+              message = @connection.dispatch_bytes(Monitor::Connection::ISMASTER_BYTES)
+              message.documents.first
+            rescue Mongo::Error
+              @connection.disconnect!
+              @connection = nil
+              raise
             end
           end
-          result
+        else
+          connection = Connection.new(server.address, options)
+          connection.connect!
+          result = server.round_trip_time_averager.measure do
+            connection.handshake!
+          end
+          @connection = connection
         end
+        result
       end
 
       # @note If the system clock is set to a time in the past, this method
       #   can sleep for a very long time.
       def throttle_scan_frequency!
-        # Normally server.last_scan indicates when the previous scan
-        # completed, but if scan! is manually invoked repeatedly then
-        # server.last_scan won't be updated and multiple scans with no
-        # cooldown can be obtained. Guard against repeated direct scan!
-        # invocation also.
-        last_time = [server.last_scan, @scan_started_at].compact.max
-        if last_time
-          difference = (Time.now - last_time)
-          throttle_time = (MIN_SCAN_FREQUENCY - difference)
-          sleep(throttle_time) if throttle_time > 0
+        delta = @next_earliest_scan - Time.now
+        if delta > 0
+          sleep(delta)
         end
-        @scan_started_at = Time.now
       end
     end
   end