mongo 2.13.0.beta1 → 2.14.0

data/lib/mongo/server/connection_base.rb

@@ -156,7 +156,9 @@ module Mongo
           operation_id = Monitoring.next_operation_id
           command_started(address, operation_id, message.payload,
             socket_object_id: socket.object_id, connection_id: id,
-            connection_generation: generation)
+            connection_generation: generation,
+            server_connection_id: description.server_connection_id,
+          )
           start = Time.now
           result = nil
           begin
@@ -184,9 +186,6 @@ module Mongo
       end
 
       def serialize(message, client, buffer = BSON::ByteBuffer.new)
-        start_size = 0
-        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
-
         # Driver specifications only mandate the fixed 16MiB limit for
         # serialized BSON documents. However, the server returns its
         # active serialized BSON document size limit in the ismaster response,
@@ -205,18 +204,45 @@ module Mongo
           if message.bulk_write?
             # Make the new maximum size equal to the specified reduced size
             # limit plus the 16KiB overhead allowance.
-            max_bson_size = REDUCED_MAX_BSON_SIZE + MAX_BSON_COMMAND_OVERHEAD
+            max_bson_size = REDUCED_MAX_BSON_SIZE
           end
-        else
-          max_bson_size += MAX_BSON_COMMAND_OVERHEAD
         end
 
-        final_message.serialize(buffer, max_bson_size)
-        if max_message_size &&
-          (buffer.length - start_size) > max_message_size
-        then
-          raise Error::MaxMessageSize.new(max_message_size)
+        # RUBY-2234: It is necessary to check that the message size does not
+        # exceed the maximum bson object size before compressing and serializing
+        # the final message.
+        #
+        # This is to avoid the case where the user performs a bulk write
+        # larger than 16MiB which, when compressed, becomes smaller than 16MiB.
+        # If the driver does not split the bulk writes prior to compression,
+        # the entire operation will be sent to the server, which will raise an
+        # error because the uncompressed operation exceeds the maximum bson size.
+        #
+        # To address this problem, we serialize the message prior to compression
+        # and raise an exception if the serialized message exceeds the maximum
+        # bson size.
+        if max_message_size
+          # Create a separate buffer that contains the un-compressed message
+          # for the purpose of checking its size. Write any pre-existing contents
+          # from the original buffer into the temporary one.
+          temp_buffer = BSON::ByteBuffer.new
+
+          # TODO: address the fact that this line mutates the buffer.
+          temp_buffer.put_bytes(buffer.get_bytes(buffer.length))
+
+          message.serialize(temp_buffer, max_bson_size, MAX_BSON_COMMAND_OVERHEAD)
+          if temp_buffer.length > max_message_size
+            raise Error::MaxMessageSize.new(max_message_size)
+          end
         end
+
+        # RUBY-2335: When the un-compressed message is smaller than the maximum
+        # bson size limit, the message will be serialized twice. The operations
+        # layer should be refactored to allow compression on an already-
+        # serialized message.
+        final_message = message.maybe_compress(compressor, options[:zlib_compression_level])
+        final_message.serialize(buffer, max_bson_size, MAX_BSON_COMMAND_OVERHEAD)
+
         buffer
       end
     end

data/lib/mongo/server/connection_common.rb

@@ -105,8 +105,8 @@ module Mongo
           options.select { |k, v| k.to_s.start_with?('ssl') }
         else
           # Due to the way options are propagated from the client, if we
-          # decide that we don't want to use TLS we need to have the ssl
-          # options explicitly set to false or the value provided to the
+          # decide that we don't want to use TLS we need to have the :ssl
+          # option explicitly set to false or the value provided to the
           # connection might be overwritten by the default inherited from
           # the client.
           {ssl: false}

data/lib/mongo/server/connection_pool.rb

@@ -766,6 +766,9 @@ module Mongo
           connection.disconnect!(reason: :error)
           raise
         end
+      rescue Error::SocketError, Error::SocketTimeoutError => exc
+        @server.unknown!(generation: exc.generation, stop_push_monitor: true)
+        raise
       end
 
       def check_invariants

data/lib/mongo/server/description.rb

@@ -182,13 +182,20 @@ module Mongo
       # @since 2.5.0
       LOGICAL_SESSION_TIMEOUT_MINUTES = 'logicalSessionTimeoutMinutes'.freeze
 
+      # Constant for reading connectionId info from config.
+      #
+      # @api private
+      CONNECTION_ID = 'connectionId'.freeze
+
       # Fields to exclude when comparing two descriptions.
       #
       # @since 2.0.6
       EXCLUDE_FOR_COMPARISON = [ LOCAL_TIME,
                                  LAST_WRITE,
                                  OPERATION_TIME,
-                                 Operation::CLUSTER_TIME ].freeze
+                                 Operation::CLUSTER_TIME,
+                                 CONNECTION_ID,
+                               ].freeze
 
       # Instantiate the new server description from the result of the ismaster
       # command.
@@ -755,6 +762,11 @@ module Mongo
       # @since 2.7.0
       attr_reader :last_update_time
 
+      # @api experimental
+      def server_connection_id
+        config['connectionId']
+      end
+
       # Check equality of two descriptions.
       #
       # @example Check description equality.

data/lib/mongo/server/monitor.rb

@@ -77,7 +77,9 @@ module Mongo
         @monitoring = monitoring
         @options = options.freeze
         @mutex = Mutex.new
+        @sdam_mutex = Mutex.new
         @next_earliest_scan = @next_wanted_scan = Time.now
+        @update_mutex = Mutex.new
       end
 
       # @return [ Server ] server The server that this monitor is monitoring.
@@ -109,6 +111,14 @@ module Mongo
       # @return [ Monitoring ] monitoring The monitoring.
       attr_reader :monitoring
 
+      # @return [ Server::PushMonitor | nil ] The push monitor, if one is being
+      #   used.
+      def push_monitor
+        @update_mutex.synchronize do
+          @push_monitor
+        end
+      end
+
       # Runs the server monitor. Refreshing happens on a separate thread per
       # server.
       #
@@ -120,9 +130,19 @@ module Mongo
       # @since 2.0.0
       def do_work
         scan!
-        delta = @next_wanted_scan - Time.now
-        if delta > 0
-          server.scan_semaphore.wait(delta)
+        # @next_wanted_scan may be updated by the push monitor.
+        # However we need to check for termination flag so that the monitor
+        # thread exits when requested.
+        loop do
+          delta = @next_wanted_scan - Time.now
+          if delta > 0
+            signaled = server.scan_semaphore.wait(delta)
+            if signaled || @stop_requested
+              break
+            end
+          else
+            break
+          end
         end
       end
 
@@ -133,6 +153,8 @@ module Mongo
       #
       # @api public for backwards compatibility only
       def stop!
+        stop_push_monitor!
+
         # Forward super's return value
         super.tap do
           # Important: disconnect should happen after the background thread
@@ -141,6 +163,15 @@ module Mongo
         end
       end
 
+      def stop_push_monitor!
+        @update_mutex.synchronize do
+          if @push_monitor
+            @push_monitor.stop!
+            @push_monitor = nil
+          end
+        end
+      end
+
       # Perform a check of the server with throttling, and update
       # the server's description and average round trip time.
       #
@@ -168,19 +199,27 @@ module Mongo
 
           result = do_scan
 
+          run_sdam_flow(result)
+        end
+      end
+
+      def run_sdam_flow(result, awaited: false)
+        @sdam_mutex.synchronize do
           old_description = server.description
 
           new_description = Description.new(server.address, result,
             server.round_trip_time_averager.average_round_trip_time)
 
-          server.cluster.run_sdam_flow(server.description, new_description)
+          server.cluster.run_sdam_flow(server.description, new_description, awaited: awaited)
 
           server.description.tap do |new_description|
-            if new_description.unknown? && !old_description.unknown?
-              @next_earliest_scan = @next_wanted_scan = Time.now
-            else
-              @next_earliest_scan = Time.now + MIN_SCAN_INTERVAL
-              @next_wanted_scan = Time.now + heartbeat_interval
+            unless awaited
+              if new_description.unknown? && !old_description.unknown?
+                @next_earliest_scan = @next_wanted_scan = Time.now
+              else
+                @next_earliest_scan = Time.now + MIN_SCAN_INTERVAL
+                @next_wanted_scan = Time.now + heartbeat_interval
+              end
             end
           end
         end
@@ -209,44 +248,19 @@ module Mongo
       end
 
       def do_scan
-        if monitoring.monitoring?
-          monitoring.started(
-            Monitoring::SERVER_HEARTBEAT,
-            Monitoring::Event::ServerHeartbeatStarted.new(server.address)
-          )
-        end
-
-        # The duration we publish in heartbeat succeeded/failed events is
-        # the time spent on the entire heartbeat. This could include time
-        # to connect the socket (including TLS handshake), not just time
-        # spent on ismaster call itself.
-        # The spec at https://github.com/mongodb/specifications/blob/master/source/server-discovery-and-monitoring/server-discovery-and-monitoring-monitoring.rst
-        # requires that the duration exposed here start from "sending the
-        # message" (ismaster). This requirement does not make sense if,
-        # for example, we were never able to connect to the server at all
-        # and thus ismaster was never sent.
-        start_time = Time.now
-
         begin
-          result = ismaster
-        rescue => exc
-          log_warn("Error running ismaster on #{server.address}: #{exc.class}: #{exc}:\n#{exc.backtrace[0..5].join("\n")}")
-          if monitoring.monitoring?
-            monitoring.failed(
-              Monitoring::SERVER_HEARTBEAT,
-              Monitoring::Event::ServerHeartbeatFailed.new(server.address, Time.now-start_time, exc)
-            )
-          end
-          result = {}
-        else
-          if monitoring.monitoring?
-            monitoring.succeeded(
-              Monitoring::SERVER_HEARTBEAT,
-              Monitoring::Event::ServerHeartbeatSucceeded.new(server.address, Time.now-start_time)
-            )
+          monitoring.publish_heartbeat(server) do
+            ismaster
           end
+        rescue => exc
+          msg = "Error running ismaster on #{server.address}"
+          Utils.warn_bg_exception(msg, exc,
+            logger: options[:logger],
+            log_prefix: options[:log_prefix],
+            bg_error_backtrace: options[:bg_error_backtrace],
+          )
+          {}
         end
-        result
       end
 
       def ismaster
@@ -274,6 +288,24 @@ module Mongo
             connection.handshake!
           end
           @connection = connection
+          if result['topologyVersion']
+            # Successful response, server 4.4+
+            @update_mutex.synchronize do
+              @push_monitor ||= PushMonitor.new(
+                self,
+                TopologyVersion.new(result['topologyVersion']),
+                monitoring,
+                **Utils.shallow_symbolize_keys(options.merge(
+                  socket_timeout: heartbeat_interval + connection.socket_timeout,
+                )),
+              )
+            end
+            push_monitor.run!
+          else
+            # Failed response or pre-4.4 server
+            stop_push_monitor!
+          end
+          result
         end
         result
       end

data/lib/mongo/server/monitor/connection.rb

@@ -118,9 +118,11 @@ module Mongo
           options[:connect_timeout] || Server::CONNECT_TIMEOUT
         end
 
+        attr_reader :server_connection_id
+
         # Sends a message and returns the result.
         #
-        # @param [ Protocol::Message ] The message to send.
+        # @param [ Protocol::Message ] message The message to send.
         #
         # @return [ Protocol::Message ] The result.
         def dispatch(message)
@@ -129,17 +131,45 @@ module Mongo
 
         # Sends a preserialized message and returns the result.
         #
-        # @param [ String ] The serialized message to send.
+        # @param [ String ] bytes The serialized message to send.
+        #
+        # @option opts [ Numeric ] :read_socket_timeout The timeout to use for
+        #   each read operation.
         #
         # @return [ Protocol::Message ] The result.
-        def dispatch_bytes(bytes)
+        def dispatch_bytes(bytes, **opts)
+          write_bytes(bytes)
+          read_response(
+            socket_timeout: opts[:read_socket_timeout],
+          )
+        end
+
+        def write_bytes(bytes)
           unless connected?
             raise ArgumentError, "Trying to dispatch on an unconnected connection #{self}"
           end
 
-          add_server_diagnostics do
-            socket.write(bytes)
-            Protocol::Message.deserialize(socket)
+          add_server_connection_id do
+            add_server_diagnostics do
+              socket.write(bytes)
+            end
+          end
+        end
+
+        # @option opts [ Numeric ] :socket_timeout The timeout to use for
+        #   each read operation.
+        def read_response(**opts)
+          unless connected?
+            raise ArgumentError, "Trying to read on an unconnected connection #{self}"
+          end
+
+          add_server_connection_id do
+            add_server_diagnostics do
+              Protocol::Message.deserialize(socket,
+                Protocol::Message::MAX_MESSAGE_SIZE,
+                nil,
+                **opts)
+            end
           end
         end
 
@@ -184,7 +214,7 @@ module Mongo
         # @since 2.0.0
         def disconnect!(options = nil)
           if socket
-            socket.close
+            socket.close rescue nil
             @socket = nil
           end
           true
@@ -200,11 +230,29 @@ module Mongo
           message = dispatch_bytes(payload)
           reply = message.documents.first
           set_compressor!(reply)
+          @server_connection_id = reply['connectionId']
           reply
-        rescue => e
-          log_warn("Failed to handshake with #{address}: #{e.class}: #{e}:\n#{e.backtrace[0..5].join("\n")}")
+        rescue => exc
+          msg = "Failed to handshake with #{address}"
+          Utils.warn_bg_exception(msg, exc,
+            logger: options[:logger],
+            log_prefix: options[:log_prefix],
+            bg_error_backtrace: options[:bg_error_backtrace],
+          )
           raise
         end
+
+        private
+
+        def add_server_connection_id
+          yield
+        rescue Mongo::Error => e
+          if server_connection_id
+            note = "sconn:#{server_connection_id}"
+            e.add_note(note)
+          end
+          raise e
+        end
       end
     end
   end

data/lib/mongo/server/pending_connection.rb

@@ -116,8 +116,13 @@ module Mongo
                 Protocol::Message.deserialize(socket, Protocol::Message::MAX_MESSAGE_SIZE).documents.first
               end
             end
-          rescue => e
-            log_warn("Failed to handshake with #{address}: #{e.class}: #{e}:\n#{e.backtrace[0..5].join("\n")}")
+          rescue => exc
+            msg = "Failed to handshake with #{address}"
+            Utils.warn_bg_exception(msg, exc,
+              logger: options[:logger],
+              log_prefix: options[:log_prefix],
+              bg_error_backtrace: options[:bg_error_backtrace],
+            )
             raise
           end
         end
@@ -151,8 +156,13 @@ module Mongo
                 speculative_auth_result: speculative_auth_result,
               )
               auth.login
-            rescue => e
-              log_warn("Failed to authenticate to #{address}: #{e.class}: #{e}:\n#{e.backtrace[0..5].join("\n")}")
+            rescue => exc
+              msg = "Failed to authenticate to #{address}"
+              Utils.warn_bg_exception(msg, exc,
+                logger: options[:logger],
+                log_prefix: options[:log_prefix],
+                bg_error_backtrace: options[:bg_error_backtrace],
+              )
               raise
             end
           end