mongo 2.8.0 → 2.9.0.rc0

data/lib/mongo/operation/update/legacy/result.rb

@@ -22,6 +22,7 @@ module Mongo
         #
         # @since 2.0.0
         class Result < Operation::Result
+          include Operation::Result::UseLegacyErrorParser
 
           # Whether an existing document was updated.
           #

data/lib/mongo/protocol/compressed.rb

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'zlib'
-
 module Mongo
   module Protocol
 

data/lib/mongo/protocol/msg.rb

@@ -31,6 +31,12 @@ module Mongo
       # @since 2.5.0
       DATABASE_IDENTIFIER = '$db'.freeze
 
+      # Keys that the driver adds to commands. These are going to be
+      # moved to the end of the hash for better logging.
+      #
+      # @api private
+      INTERNAL_KEYS = Set.new(%w($clusterTime lsid signature txnNumber)).freeze
+
       # Creates a new OP_MSG protocol message
       #
       # @example Create a OP_MSG wire protocol message
@@ -82,10 +88,27 @@ module Mongo
       #
       # @since 2.5.0
       def payload
+        # Reorder keys in global_args for better logging - see
+        # https://jira.mongodb.org/browse/RUBY-1591.
+        # Note that even without the reordering, the payload is not an exact
+        # match to what is sent over the wire because the command as used in
+        # the published eent combines keys from multiple sections of the
+        # payload sent over the wire.
+        ordered_command = {}
+        skipped_command = {}
+        command.each do |k, v|
+          if INTERNAL_KEYS.member?(k.to_s)
+            skipped_command[k] = v
+          else
+            ordered_command[k] = v
+          end
+        end
+        ordered_command.update(skipped_command)
+
         BSON::Document.new(
-          command_name: command.keys.first.to_s,
+          command_name: ordered_command.keys.first.to_s,
           database_name: global_args[DATABASE_IDENTIFIER],
-          command: command,
+          command: ordered_command,
           request_id: request_id,
           reply: sections[0]
         )

data/lib/mongo/retryable.rb

@@ -19,50 +19,115 @@ module Mongo
   # @since 2.1.0
   module Retryable
 
-    # Execute a read operation with a retry.
+    # Execute a read operation returning a cursor with retrying.
+    #
+    # This method performs server selection for the specified server selector
+    # and yields to the provided block, which should execute the initial
+    # query operation and return its result. The block will be passed the
+    # server selected for the operation. If the block raises an exception,
+    # and this exception corresponds to a read retryable error, and read
+    # retries are enabled for the client, this method will perform server
+    # selection again and yield to the block again (with potentially a
+    # different server). If the block returns successfully, the result
+    # of the block (which should be a Mongo::Operation::Result) is used to
+    # construct a Mongo::Cursor object for the result set. The cursor
+    # is then returned.
+    #
+    # If modern retry reads are on (which is the default), the initial read
+    # operation will be retried once. If legacy retry reads are on, the
+    # initial read operation will be retried zero or more times depending
+    # on the :max_read_retries client setting, the default for which is 1.
+    # To disable read retries, turn off modern read retries by setting
+    # retry_reads: false and set :max_read_retries to 0 on the client.
     #
     # @api private
     #
-    # @example Execute the read.
-    #   read_with_retry do
+    # @example Execute a read returning a cursor.
+    #   cursor = read_with_retry_cursor(session, server_selector, view) do |server|
+    #     # return a Mongo::Operation::Result
     #     ...
     #   end
     #
-    # @note This only retries read operations on socket errors.
+    # @param [ Mongo::Session ] session The session that the operation is being
+    #   run on.
+    # @param [ Mongo::ServerSelector::Selectable ] server_selector Server
+    #   selector for the operation.
+    # @param [ CollectionView ] view The +CollectionView+ defining the query.
+    # @param [ Proc ] block The block to execute.
+    #
+    # @return [ Cursor ] The cursor for the result set.
+    def read_with_retry_cursor(session, server_selector, view, &block)
+      read_with_retry(session, server_selector) do |server|
+        result = yield server
+        Cursor.new(view, result, server, session: session)
+      end
+    end
+
+    # Execute a read operation with retrying.
+    #
+    # This method performs server selection for the specified server selector
+    # and yields to the provided block, which should execute the initial
+    # query operation and return its result. The block will be passed the
+    # server selected for the operation. If the block raises an exception,
+    # and this exception corresponds to a read retryable error, and read
+    # retries are enabled for the client, this method will perform server
+    # selection again and yield to the block again (with potentially a
+    # different server). If the block returns successfully, the result
+    # of the block is returned.
     #
-    # @param [ Mongo::Session ] session The session that the operation is being run on.
+    # If modern retry reads are on (which is the default), the initial read
+    # operation will be retried once. If legacy retry reads are on, the
+    # initial read operation will be retried zero or more times depending
+    # on the :max_read_retries client setting, the default for which is 1.
+    # To disable read retries, turn off modern read retries by setting
+    # retry_reads: false and set :max_read_retries to 0 on the client.
+    #
+    # @api private
+    #
+    # @example Execute the read.
+    #   read_with_retry(session, server_selector) do |server|
+    #     ...
+    #   end
+    #
+    # @param [ Mongo::Session ] session The session that the operation is being
+    #   run on.
+    # @param [ Mongo::ServerSelector::Selectable ] server_selector Server
+    #   selector for the operation.
     # @param [ Proc ] block The block to execute.
     #
     # @return [ Result ] The result of the operation.
-    #
-    # @since 2.1.0
-    def read_with_retry(session = nil)
-      attempt = 0
-      begin
-        attempt += 1
-        yield
-      rescue Error::SocketError, Error::SocketTimeoutError => e
-        if attempt > cluster.max_read_retries || (session && session.in_transaction?)
-          raise
-        end
-        log_retry(e)
-        cluster.scan!(false)
-        retry
-      rescue Error::OperationFailure => e
-        if cluster.sharded? && e.retryable? && !(session && session.in_transaction?)
-          if attempt > cluster.max_read_retries
-            raise
-          end
-          log_retry(e)
-          sleep(cluster.read_retry_interval)
-          retry
-        else
-          raise
+    def read_with_retry(session = nil, server_selector = nil, &block)
+      if session.nil? && server_selector.nil?
+        # Older versions of Mongoid call read_with_retry without arguments.
+        # This is already not correct in a MongoDB 3.6+ environment with
+        # sessions. For compatibility we emulate the legacy driver behavior
+        # here but upgrading Mongoid is strongly recommended.
+        unless $_mongo_read_with_retry_warned
+          $_mongo_read_with_retry_warned = true
+          Logger.logger.warn("Legacy read_with_retry invocation - please update the application and/or its dependencies")
         end
+        # Since we don't have a session, we cannot use the modern read retries.
+        # And we need to select a server but we don't have a server selector.
+        # Use PrimaryPreferred which will work as long as there is a data
+        # bearing node in the cluster; the block may select a different server
+        # which is fine.
+        server_selector = ServerSelector.get(mode: :primary_preferred)
+        legacy_read_with_retry(nil, server_selector, &block)
+      elsif session && session.retry_reads?
+        modern_read_with_retry(session, server_selector, &block)
+      elsif client.max_read_retries > 0
+        legacy_read_with_retry(session, server_selector, &block)
+      else
+        server = select_server(cluster, server_selector)
+        yield server
       end
     end
 
-    # Execute a read operation with a single retry.
+    # Execute a read operation with a single retry on network errors.
+    #
+    # This method is used by the driver for some of the internal housekeeping
+    # operations. Application-requested reads should use read_with_retry
+    # rather than this method.
     #
     # @api private
     #
@@ -159,6 +224,52 @@ module Mongo
 
     private
 
+    def modern_read_with_retry(session, server_selector, &block)
+      attempt = 0
+      server = select_server(cluster, server_selector)
+      begin
+        yield server
+      rescue Error::SocketError, Error::SocketTimeoutError => e
+        if session.in_transaction?
+          raise
+        end
+        retry_read(e, server_selector, &block)
+      rescue Error::OperationFailure => e
+        if session.in_transaction? || !e.write_retryable?
+          raise
+        end
+        retry_read(e, server_selector, &block)
+      end
+    end
+
+    def legacy_read_with_retry(session, server_selector)
+      attempt = 0
+      server = select_server(cluster, server_selector)
+      begin
+        attempt += 1
+        yield server
+      rescue Error::SocketError, Error::SocketTimeoutError => e
+        if attempt > client.max_read_retries || (session && session.in_transaction?)
+          raise
+        end
+        log_retry(e, message: 'Legacy read retry')
+        server = select_server(cluster, server_selector)
+        retry
+      rescue Error::OperationFailure => e
+        if cluster.sharded? && e.retryable? && !(session && session.in_transaction?)
+          if attempt > client.max_read_retries
+            raise
+          end
+          log_retry(e, message: 'Legacy read retry')
+          sleep(client.read_retry_interval)
+          server = select_server(cluster, server_selector)
+          retry
+        else
+          raise
+        end
+      end
+    end
+
     def retry_write_allowed?(session, write_concern)
       unless session && session.retry_writes?
         return false
@@ -174,6 +285,27 @@ module Mongo
       end
     end
 
+    def retry_read(original_error, server_selector, &block)
+      begin
+        server = select_server(cluster, server_selector)
+      rescue
+        raise original_error
+      end
+
+      log_retry(original_error, message: 'Read retry')
+
+      begin
+        yield server, true
+      rescue Error::SocketError, Error::SocketTimeoutError => e
+        raise e
+      rescue Error::OperationFailure => e
+        raise original_error unless e.write_retryable?
+        raise e
+      rescue
+        raise original_error
+      end
+    end
+
     def retry_write(original_error, txn_num, &block)
       # We do not request a scan of the cluster here, because error handling
       # for the error which triggered the retry should have updated the
@@ -182,7 +314,7 @@ module Mongo
       # server unknown). Here we just need to wait for server selection.
       server = cluster.next_primary
       raise original_error unless (server.retry_writes? && txn_num)
-      log_retry(original_error)
+      log_retry(original_error, message: 'Write retry')
       yield(server, txn_num, true)
     rescue Error::SocketError, Error::SocketTimeoutError => e
       raise e
@@ -203,11 +335,11 @@ module Mongo
         yield(server || cluster.next_primary)
       rescue Error::OperationFailure => e
         server = nil
-        if attempt > Cluster::MAX_WRITE_RETRIES
+        if attempt > client.max_write_retries
           raise
         end
         if e.write_retryable? && !(session && session.in_transaction?)
-          log_retry(e)
+          log_retry(e, message: 'Legacy write retry')
           cluster.scan!(false)
           retry
         else
@@ -216,6 +348,12 @@ module Mongo
       end
     end
 
+    # This is a separate method to make it possible for the test suite to
+    # assert that server selection is performed during retry attempts.
+    def select_server(cluster, server_selector)
+      server_selector.select_server(cluster)
+    end
+
     # Log a warning so that any application slow down is immediately obvious.
     def log_retry(e, options = nil)
       message = if options && options[:message]

data/lib/mongo/server.rb

@@ -61,6 +61,9 @@ module Mongo
       monitor = options.delete(:monitor)
       @options = options.freeze
       @event_listeners = event_listeners
+      @connection_id_gen = Class.new do
+        include Id
+      end
       @monitor = Monitor.new(address, event_listeners, monitoring,
         options.merge(app_metadata: Monitor::AppMetadata.new(cluster.options)))
       unless monitor == false
@@ -178,7 +181,13 @@ module Mongo
     #
     # @since 2.0.0
     def disconnect!(wait=false)
-      pool.disconnect!
+      begin
+        # For backwards compatibility we disconnect/clear the pool rather
+        # than close it here.
+        pool.disconnect!
+      rescue Error::PoolClosedError
+        # If the pool was already closed, we don't need to do anything here.
+      end
       monitor.stop!(wait)
       @connected = false
       true
@@ -282,11 +291,7 @@ module Mongo
     # @since 2.0.0
     def pool
       @pool_lock.synchronize do
-        @pool ||= begin
-          ConnectionPool.new(options) do |generation|
-            Connection.new(self, options.merge(generation: generation))
-          end
-        end
+        @pool ||= ConnectionPool.new(self, options)
       end
     end
 
@@ -315,7 +320,9 @@ module Mongo
     #
     # @since 2.1.0
     def reconnect!
-      monitor.restart!
+      if options[:monitoring_io] != false
+        monitor.restart!
+      end
       @connected = true
     end
 
@@ -373,6 +380,13 @@ module Mongo
       raise
     end
 
+    # Whether the server supports modern read retries.
+    #
+    # @api private
+    def retry_reads?
+      !!(features.sessions_enabled? && logical_session_timeout)
+    end
+
     # Will writes sent to this server be retried.
     #
     # @example Will writes be retried.
@@ -403,6 +417,11 @@ module Mongo
     def update_description(description)
       monitor.instance_variable_set('@description', description)
     end
+
+    # @api private
+    def next_connection_id
+      @connection_id_gen.next_id
+    end
   end
 end
 

data/lib/mongo/server/app_metadata.rb

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'rbconfig'
-
 module Mongo
   class Server
     # Application metadata that is sent to the server in an ismaster command,

data/lib/mongo/server/connectable.rb

@@ -81,15 +81,21 @@ module Mongo
           result
         ensure
           unless success
-            disconnect!
+            disconnect!(reason: :error)
           end
         end
       end
 
       def ensure_same_process!
         if pid != Process.pid
-          disconnect!
+          # When we reconnect here, CMAP events won't be correctly sent
+          # since the CMAP spec does not permit a connection to be disconnected
+          # and then reconnected
+          log_warn("Detected PID change - Mongo client should have been reconnected (old pid #{pid}, new pid #{Process.pid}")
+          disconnect!(reason: :stale)
+          @closed = false
           @pid = Process.pid
+          connect!
         end
       end
     end

data/lib/mongo/server/connection.rb

@@ -89,6 +89,7 @@ module Mongo
       #
       # @since 2.0.0
       def initialize(server, options = {})
+        @id = server.next_connection_id
         @monitoring = server.monitoring
         @options = options.freeze
         @server = server
@@ -97,13 +98,23 @@ module Mongo
         @last_checkin = nil
         @auth_mechanism = nil
         @pid = Process.pid
+
+        publish_cmap_event(
+          Monitoring::Event::Cmap::ConnectionCreated.new(address, id)
+        )
       end
 
-      # The last time the connection was checked back into a pool.
+      # @return [ Time ] The last time the connection was checked back into a pool.
       #
       # @since 2.5.0
       attr_reader :last_checkin
 
+      # @return [ Integer ] The ID for the connection. This will be unique
+      # across connections to the same server object.
+      #
+      # @since 2.9.0
+      attr_reader :id
+
       # Connection pool generation from which this connection was created.
       # May be nil.
       #
@@ -113,6 +124,18 @@ module Mongo
         options[:generation]
       end
 
+      # Whether the connection was closed.
+      #
+      # Closed connections should no longer be used. Instead obtain a new
+      # connection from the connection pool.
+      #
+      # @return [ true | false ] Whether connection was closed.
+      #
+      # @since 2.9.0
+      def closed?
+        !!@closed
+      end
+
       # Establishes a network connection to the target address.
       #
       # If the connection is already established, this method does nothing.
@@ -120,44 +143,91 @@ module Mongo
       # @example Connect to the host.
       #   connection.connect!
       #
-      # @note This method mutates the connection class by setting a socket if
+      # @note This method mutates the connection object by setting a socket if
       #   one previously did not exist.
       #
       # @return [ true ] If the connection succeeded.
       #
       # @since 2.0.0
       def connect!
+        if closed?
+          if Lint.enabled?
+            raise Error::LintError, "Reconnecting closed connections is no longer supported"
+          else
+            log_warn("Reconnecting closed connections is deprecated (for #{address})")
+          end
+        end
+
         unless @socket
-          socket = address.socket(socket_timeout, ssl_options,
-            connect_timeout: address.connect_timeout)
-          handshake!(socket)
-          pending_connection = PendingConnection.new(socket, @server, monitoring, options)
-          authenticate!(pending_connection)
           # When @socket is assigned, the socket should have handshaken and
           # authenticated and be usable.
-          @socket = socket
+          @socket = do_connect
+
+          publish_cmap_event(
+            Monitoring::Event::Cmap::ConnectionReady.new(address, id)
+          )
+
+          @close_event_published = false
         end
         true
       end
 
+      # Separate method to permit easier mocking in the test suite.
+      def do_connect
+        socket = address.socket(socket_timeout, ssl_options,
+          connect_timeout: address.connect_timeout)
+        handshake!(socket)
+        pending_connection = PendingConnection.new(socket, @server, monitoring, options)
+        authenticate!(pending_connection)
+        socket
+      end
+      private :do_connect
+
       # Disconnect the connection.
       #
-      # @example Disconnect from the host.
-      #   connection.disconnect!
+      # @note Once a connection is disconnected, it should no longer be used.
+      #   A new connection should be obtained from the connection pool which
+      #   will either return a ready connection or create a new connection.
+      #   If linting is enabled, reusing a disconnected connection will raise
+      #   Error::LintError. If linting is not enabled, a warning will be logged.
+      #
+      # @note This method mutates the connection object by setting the socket
+      #   to nil if the closing succeeded.
       #
-      # @note This method mutates the connection by setting the socket to nil
-      #   if the closing succeeded.
+      # @option options [ Symbol ] :reason The reason why the connection is
+      #   being closed.
       #
       # @return [ true ] If the disconnect succeeded.
       #
       # @since 2.0.0
-      def disconnect!
+      def disconnect!(options = nil)
+        # Note: @closed may be true here but we also may have a socket.
+        # Check the socket and not @closed flag.
         @auth_mechanism = nil
         @last_checkin = nil
         if socket
           socket.close
           @socket = nil
         end
+        @closed = true
+
+        # To satisfy CMAP spec tests, publish close events even if the
+        # socket was never connected (and thus the ready event was never
+        # published). But track whether we published close event and do not
+        # publish it multiple times, unless the socket was reconnected -
+        # in that case publish the close event once per socket close.
+        unless @close_event_published
+          reason = options && options[:reason]
+          publish_cmap_event(
+            Monitoring::Event::Cmap::ConnectionClosed.new(
+              address,
+              id,
+              reason,
+            ),
+          )
+          @close_event_published = true
+        end
+
         true
       end