mongo 2.13.0.beta1 → 2.13.0.rc1

data/lib/mongo/monitoring/event/server_heartbeat_failed.rb

@@ -36,6 +36,11 @@ module Mongo
         # Alias of error for SDAM spec compliance.
         alias :failure :error
 
+        # @return [ true | false ] Whether the heartbeat was awaited.
+        def awaited?
+          @awaited
+        end
+
         # Create the event.
         #
         # @example Create the event.
@@ -43,13 +48,15 @@ module Mongo
         #
         # @param [ Address ] address The server address.
         # @param [ Float ] round_trip_time Duration of ismaster call in seconds.
+        # @param [ true | false ] awaited Whether the heartbeat was awaited.
         #
         # @since 2.7.0
         # @api private
-        def initialize(address, round_trip_time, error)
+        def initialize(address, round_trip_time, error, awaited: false)
           @address = address
           @round_trip_time = round_trip_time
           @error = error
+          @awaited = !!awaited
         end
 
         # Returns a concise yet useful summary of the event.
@@ -61,7 +68,7 @@ module Mongo
         # @since 2.7.0
         # @api experimental
         def summary
-          "#<#{self.class.name.sub(/^Mongo::Monitoring::Event::/, '')}" +
+          "#<#{short_class_name}" +
           " address=#{address}" +
           " error=#{error.inspect}>"
         end

data/lib/mongo/monitoring/event/server_heartbeat_started.rb

@@ -24,17 +24,24 @@ module Mongo
         # @return [ Address ] address The server address.
         attr_reader :address
 
+        # @return [ true | false ] Whether the heartbeat was awaited.
+        def awaited?
+          @awaited
+        end
+
         # Create the event.
         #
         # @example Create the event.
         #   ServerHeartbeatStarted.new(address)
         #
         # @param [ Address ] address The server address.
+        # @param [ true | false ] awaited Whether the heartbeat was awaited.
         #
         # @since 2.7.0
         # @api private
-        def initialize(address)
+        def initialize(address, awaited: false)
           @address = address
+          @awaited = !!awaited
         end
 
         # Returns a concise yet useful summary of the event.
@@ -46,7 +53,7 @@ module Mongo
         # @since 2.7.0
         # @api experimental
         def summary
-          "#<#{self.class.name.sub(/^Mongo::Monitoring::Event::/, '')}" +
+          "#<#{short_class_name}" +
           " address=#{address}>"
         end
       end

data/lib/mongo/monitoring/event/server_heartbeat_succeeded.rb

@@ -30,6 +30,11 @@ module Mongo
         # Alias of round_trip_time.
         alias :duration :round_trip_time
 
+        # @return [ true | false ] Whether the heartbeat was awaited.
+        def awaited?
+          @awaited
+        end
+
         # Create the event.
         #
         # @example Create the event.
@@ -37,12 +42,14 @@ module Mongo
         #
         # @param [ Address ] address The server address.
         # @param [ Float ] round_trip_time Duration of ismaster call in seconds.
+        # @param [ true | false ] awaited Whether the heartbeat was awaited.
         #
         # @since 2.7.0
         # @api private
-        def initialize(address, round_trip_time)
+        def initialize(address, round_trip_time, awaited: false)
           @address = address
           @round_trip_time = round_trip_time
+          @awaited = !!awaited
         end
 
         # Returns a concise yet useful summary of the event.
@@ -54,7 +61,7 @@ module Mongo
         # @since 2.7.0
         # @api experimental
         def summary
-          "#<#{self.class.name.sub(/^Mongo::Monitoring::Event::/, '')}" +
+          "#<#{short_class_name}" +
           " address=#{address}>"
         end
       end

data/lib/mongo/monitoring/event/server_opening.rb

@@ -50,7 +50,7 @@ module Mongo
         # @since 2.7.0
         # @api experimental
         def summary
-          "#<#{self.class.name.sub(/^Mongo::Monitoring::Event::/, '')}" +
+          "#<#{short_class_name}" +
           " address=#{address} topology=#{topology.summary}>"
         end
       end

data/lib/mongo/monitoring/event/topology_changed.rb

@@ -50,7 +50,7 @@ module Mongo
         # @since 2.7.0
         # @api experimental
         def summary
-          "#<#{self.class.name.sub(/^Mongo::Monitoring::Event::/, '')}" +
+          "#<#{short_class_name}" +
           " prev=#{previous_topology.summary}" +
           " new=#{new_topology.summary}>"
         end

data/lib/mongo/monitoring/event/topology_closed.rb

@@ -45,7 +45,7 @@ module Mongo
         # @since 2.7.0
         # @api experimental
         def summary
-          "#<#{self.class.name.sub(/^Mongo::Monitoring::Event::/, '')}" +
+          "#<#{short_class_name}" +
           " topology=#{topology.summary}>"
         end
       end

data/lib/mongo/monitoring/event/topology_opening.rb

@@ -45,7 +45,7 @@ module Mongo
         # @since 2.7.0
         # @api experimental
         def summary
-          "#<#{self.class.name.sub(/^Mongo::Monitoring::Event::/, '')}" +
+          "#<#{short_class_name}" +
           " topology=#{topology.summary}>"
         end
       end

data/lib/mongo/monitoring/publishable.rb

@@ -32,7 +32,7 @@ module Mongo
       def publish_sdam_event(topic, event)
         return unless monitoring?
 
-        log_debug("EVENT: #{event.summary}")
+        #log_debug("EVENT: #{event.summary}")
         monitoring.succeeded(topic, event)
       end
 
@@ -45,13 +45,16 @@ module Mongo
       private
 
       def command_started(address, operation_id, payload,
-        socket_object_id: nil, connection_id: nil, connection_generation: nil
+        socket_object_id: nil, connection_id: nil, connection_generation: nil,
+        server_connection_id: nil
       )
         monitoring.started(
           Monitoring::COMMAND,
           Event::CommandStarted.generate(address, operation_id, payload,
             socket_object_id: socket_object_id, connection_id: connection_id,
-            connection_generation: connection_generation)
+            connection_generation: connection_generation,
+            server_connection_id: server_connection_id,
+          )
         )
       end
 

data/lib/mongo/monitoring/server_description_changed_log_subscriber.rb

@@ -25,9 +25,17 @@ module Mongo
       def log_event(event)
         log_debug(
           "Server description for #{event.address} changed from " +
-          "'#{event.previous_description.server_type}' to '#{event.new_description.server_type}'."
+          "'#{event.previous_description.server_type}' to '#{event.new_description.server_type}'#{awaited_indicator(event)}."
         )
       end
+
+      def awaited_indicator(event)
+        if event.awaited?
+          ' [awaited]'
+        else
+          ''
+        end
+      end
     end
   end
 end

data/lib/mongo/monitoring/topology_changed_log_subscriber.rb

@@ -23,7 +23,7 @@ module Mongo
       private
 
       def log_event(event)
-        if event.previous_topology != event.new_topology
+        if event.previous_topology.class != event.new_topology.class
           log_debug(
             "Topology type '#{event.previous_topology.display_name}' changed to " +
             "type '#{event.new_topology.display_name}'."

data/lib/mongo/protocol/message.rb

@@ -199,10 +199,33 @@ module Mongo
       # @option options [ Boolean ] :deserialize_as_bson Whether to deserialize
       #   this message using BSON types instead of native Ruby types wherever
       #   possible.
+      # @option options [ Numeric ] :socket_timeout The timeout to use for
+      #   each read operation.
       #
       # @return [ Message ] Instance of a Message class
-      def self.deserialize(io, max_message_size = MAX_MESSAGE_SIZE, expected_response_to = nil, options = {})
-        length, _request_id, response_to, _op_code = deserialize_header(BSON::ByteBuffer.new(io.read(16)))
+      #
+      # @api private
+      def self.deserialize(io,
+        max_message_size = MAX_MESSAGE_SIZE,
+        expected_response_to = nil,
+        options = {}
+      )
+        # io is usually a Mongo::Socket instance, which supports the
+        # timeout option. For compatibility with whoever might call this
+        # method with some other IO-like object, pass options only when they
+        # are not empty.
+        read_options = {}
+        if timeout = options[:socket_timeout]
+          read_options[:timeout] = timeout
+        end
+
+        if read_options.empty?
+          chunk = io.read(16)
+        else
+          chunk = io.read(16, **read_options)
+        end
+        buf = BSON::ByteBuffer.new(chunk)
+        length, _request_id, response_to, _op_code = deserialize_header(buf)
 
         # Protection from potential DOS man-in-the-middle attacks. See
         # DRIVERS-276.
@@ -216,14 +239,19 @@ module Mongo
           raise Error::UnexpectedResponse.new(expected_response_to, response_to)
         end
 
-        message = Registry.get(_op_code).allocate
-        buffer = BSON::ByteBuffer.new(io.read(length - 16))
+        if read_options.empty?
+          chunk = io.read(length - 16)
+        else
+          chunk = io.read(length - 16, **read_options)
+        end
+        buf = BSON::ByteBuffer.new(chunk)
 
+        message = Registry.get(_op_code).allocate
         message.send(:fields).each do |field|
           if field[:multi]
-            deserialize_array(message, buffer, field, options)
+            deserialize_array(message, buf, field, options)
           else
-            deserialize_field(message, buffer, field, options)
+            deserialize_field(message, buf, field, options)
           end
         end
         if message.is_a?(Msg)
@@ -374,7 +402,7 @@ module Mongo
       #   each of the elements in this array using BSON types wherever possible.
       #
       # @return [Message] Message with deserialized array.
-      def self.deserialize_array(message, io, field, options)
+      def self.deserialize_array(message, io, field, options = {})
         elements = []
         count = message.instance_variable_get(field[:multi])
         count.times { elements << field[:type].deserialize(io, options) }
@@ -392,7 +420,7 @@ module Mongo
       #   this field using BSON types wherever possible.
       #
       # @return [Message] Message with deserialized field.
-      def self.deserialize_field(message, io, field, options)
+      def self.deserialize_field(message, io, field, options = {})
         message.instance_variable_set(
           field[:name],
           field[:type].deserialize(io, options)

data/lib/mongo/protocol/msg.rb

@@ -60,6 +60,13 @@ module Mongo
       #
       # @since 2.5.0
       def initialize(flags, options, main_document, *sequences)
+        if flags
+          flags.each do |flag|
+            unless KNOWN_FLAGS.key?(flag)
+              raise ArgumentError, "Unknown flag: #{flag.inspect}"
+            end
+          end
+        end
         @flags = flags || []
         @options = options
         unless main_document.is_a?(Hash)
@@ -313,10 +320,17 @@ module Mongo
       # @since 2.5.0
       OP_CODE = 2013
 
+      KNOWN_FLAGS = {
+        checksum_present: true,
+        more_to_come: true,
+        exhaust_allowed: true,
+      }
+
       # Available flags for a OP_MSG message.
       FLAGS = Array.new(16).tap do |arr|
         arr[0] = :checksum_present
         arr[1] = :more_to_come
+        arr[16] = :exhaust_allowed
       end.freeze
 
       # @!attribute

data/lib/mongo/protocol/serializers.rb

@@ -440,12 +440,15 @@ module Mongo
         # Deserializes bytes from the byte buffer.
         #
         # @param [ BSON::ByteBuffer ] buffer Buffer containing the value to read.
-        # @param [ Integer ] num_bytes Number of bytes to read.
+        # @param [ Hash ] options The method options.
+        #
+        # @option options [ Integer ] num_bytes Number of bytes to read.
         #
         # @return [ String ] The bytes.
         #
         # @since 2.5.0
-        def self.deserialize(buffer, num_bytes = nil)
+        def self.deserialize(buffer, options = {})
+          num_bytes = options[:num_bytes]
           buffer.get_bytes(num_bytes || buffer.length)
         end
       end

data/lib/mongo/server.rb

@@ -64,7 +64,7 @@ module Mongo
       @connection_id_gen = Class.new do
         include Id
       end
-      @scan_semaphore = Semaphore.new
+      @scan_semaphore = DistinguishingSemaphore.new
       @round_trip_time_averager = RoundTripTimeAverager.new
       @description = Description.new(address, {})
       @last_scan = nil
@@ -432,7 +432,7 @@ module Mongo
     def handle_handshake_failure!
       yield
     rescue Mongo::Error::SocketError, Mongo::Error::SocketTimeoutError => e
-      unknown!(generation: e.generation)
+      unknown!(generation: e.generation, stop_push_monitor: true)
       raise
     end
 
@@ -455,7 +455,7 @@ module Mongo
       raise
     rescue Mongo::Error::SocketError => e
       # non-timeout network error
-      unknown!(generation: e.generation)
+      unknown!(generation: e.generation, stop_push_monitor: true)
       raise
     rescue Auth::Unauthorized
       # auth error, keep server description and topology as they are
@@ -503,6 +503,8 @@ module Mongo
     #   on 4.2+ servers).
     # @option options [ TopologyVersion ] :topology_version Topology version
     #   of the error response that is causing the server to be marked unknown.
+    # @option options [ true | false ] :stop_push_monitor Whether to stop
+    #   the PushMonitor associated with the server, if any.
     #
     # @since 2.4.0, SDAM events are sent as of version 2.7.0
     def unknown!(options = {})
@@ -516,6 +518,10 @@ module Mongo
         return
       end
 
+      if options[:stop_push_monitor]
+        monitor&.stop_push_monitor!
+      end
+
       # SDAM flow will update description on the server without in-place
       # mutations and invoke SDAM transitions as needed.
       config = {}
@@ -562,3 +568,4 @@ require 'mongo/server/context'
 require 'mongo/server/description'
 require 'mongo/server/monitor'
 require 'mongo/server/round_trip_time_averager'
+require 'mongo/server/push_monitor'

data/lib/mongo/server/connection.rb

@@ -163,7 +163,7 @@ module Mongo
         unless @socket
           # When @socket is assigned, the socket should have handshaken and
           # authenticated and be usable.
-          @socket, @description = do_connect
+          @socket, @description, @compressor = do_connect
 
           publish_cmap_event(
             Monitoring::Event::Cmap::ConnectionReady.new(address, id)
@@ -194,7 +194,7 @@ module Mongo
           raise
         end
 
-        [socket, pending_connection.description]
+        [socket, pending_connection.description, pending_connection.compressor]
       end
 
       # Disconnect the connection.
@@ -220,7 +220,7 @@ module Mongo
         @auth_mechanism = nil
         @last_checkin = nil
         if socket
-          socket.close
+          socket.close rescue nil
           @socket = nil
         end
         @closed = true
@@ -306,7 +306,7 @@ module Mongo
           yield
         rescue Error::SocketError => e
           @error = e
-          @server.unknown!(generation: e.generation)
+          @server.unknown!(generation: e.generation, stop_push_monitor: true)
           raise
         rescue Error::SocketTimeoutError => e
           @error = e

data/lib/mongo/server/connection_base.rb

@@ -156,7 +156,9 @@ module Mongo
           operation_id = Monitoring.next_operation_id
           command_started(address, operation_id, message.payload,
             socket_object_id: socket.object_id, connection_id: id,
-            connection_generation: generation)
+            connection_generation: generation,
+            server_connection_id: description.server_connection_id,
+          )
           start = Time.now
           result = nil
           begin

data/lib/mongo/server/description.rb

@@ -755,6 +755,11 @@ module Mongo
       # @since 2.7.0
       attr_reader :last_update_time
 
+      # @api experimental
+      def server_connection_id
+        config['connectionId']
+      end
+
       # Check equality of two descriptions.
       #
       # @example Check description equality.

data/lib/mongo/server/monitor.rb

@@ -77,7 +77,9 @@ module Mongo
         @monitoring = monitoring
         @options = options.freeze
         @mutex = Mutex.new
+        @sdam_mutex = Mutex.new
         @next_earliest_scan = @next_wanted_scan = Time.now
+        @update_mutex = Mutex.new
       end
 
       # @return [ Server ] server The server that this monitor is monitoring.
@@ -109,6 +111,14 @@ module Mongo
       # @return [ Monitoring ] monitoring The monitoring.
       attr_reader :monitoring
 
+      # @return [ Server::PushMonitor | nil ] The push monitor, if one is being
+      #   used.
+      def push_monitor
+        @update_mutex.synchronize do
+          @push_monitor
+        end
+      end
+
       # Runs the server monitor. Refreshing happens on a separate thread per
       # server.
       #
@@ -120,9 +130,19 @@ module Mongo
       # @since 2.0.0
       def do_work
         scan!
-        delta = @next_wanted_scan - Time.now
-        if delta > 0
-          server.scan_semaphore.wait(delta)
+        # @next_wanted_scan may be updated by the push monitor.
+        # However we need to check for termination flag so that the monitor
+        # thread exits when requested.
+        loop do
+          delta = @next_wanted_scan - Time.now
+          if delta > 0
+            signaled = server.scan_semaphore.wait(delta)
+            if signaled || @stop_requested
+              break
+            end
+          else
+            break
+          end
         end
       end
 
@@ -133,6 +153,8 @@ module Mongo
       #
       # @api public for backwards compatibility only
       def stop!
+        stop_push_monitor!
+
         # Forward super's return value
         super.tap do
           # Important: disconnect should happen after the background thread
@@ -141,6 +163,15 @@ module Mongo
         end
       end
 
+      def stop_push_monitor!
+        @update_mutex.synchronize do
+          if @push_monitor
+            @push_monitor.stop!
+            @push_monitor = nil
+          end
+        end
+      end
+
       # Perform a check of the server with throttling, and update
       # the server's description and average round trip time.
       #
@@ -168,19 +199,27 @@ module Mongo
 
           result = do_scan
 
+          run_sdam_flow(result)
+        end
+      end
+
+      def run_sdam_flow(result, awaited: false)
+        @sdam_mutex.synchronize do
           old_description = server.description
 
           new_description = Description.new(server.address, result,
             server.round_trip_time_averager.average_round_trip_time)
 
-          server.cluster.run_sdam_flow(server.description, new_description)
+          server.cluster.run_sdam_flow(server.description, new_description, awaited: awaited)
 
           server.description.tap do |new_description|
-            if new_description.unknown? && !old_description.unknown?
-              @next_earliest_scan = @next_wanted_scan = Time.now
-            else
-              @next_earliest_scan = Time.now + MIN_SCAN_INTERVAL
-              @next_wanted_scan = Time.now + heartbeat_interval
+            unless awaited
+              if new_description.unknown? && !old_description.unknown?
+                @next_earliest_scan = @next_wanted_scan = Time.now
+              else
+                @next_earliest_scan = Time.now + MIN_SCAN_INTERVAL
+                @next_wanted_scan = Time.now + heartbeat_interval
+              end
             end
           end
         end
@@ -209,44 +248,19 @@ module Mongo
       end
 
       def do_scan
-        if monitoring.monitoring?
-          monitoring.started(
-            Monitoring::SERVER_HEARTBEAT,
-            Monitoring::Event::ServerHeartbeatStarted.new(server.address)
-          )
-        end
-
-        # The duration we publish in heartbeat succeeded/failed events is
-        # the time spent on the entire heartbeat. This could include time
-        # to connect the socket (including TLS handshake), not just time
-        # spent on ismaster call itself.
-        # The spec at https://github.com/mongodb/specifications/blob/master/source/server-discovery-and-monitoring/server-discovery-and-monitoring-monitoring.rst
-        # requires that the duration exposed here start from "sending the
-        # message" (ismaster). This requirement does not make sense if,
-        # for example, we were never able to connect to the server at all
-        # and thus ismaster was never sent.
-        start_time = Time.now
-
         begin
-          result = ismaster
-        rescue => exc
-          log_warn("Error running ismaster on #{server.address}: #{exc.class}: #{exc}:\n#{exc.backtrace[0..5].join("\n")}")
-          if monitoring.monitoring?
-            monitoring.failed(
-              Monitoring::SERVER_HEARTBEAT,
-              Monitoring::Event::ServerHeartbeatFailed.new(server.address, Time.now-start_time, exc)
-            )
-          end
-          result = {}
-        else
-          if monitoring.monitoring?
-            monitoring.succeeded(
-              Monitoring::SERVER_HEARTBEAT,
-              Monitoring::Event::ServerHeartbeatSucceeded.new(server.address, Time.now-start_time)
-            )
+          monitoring.publish_heartbeat(server) do
+            ismaster
           end
+        rescue => exc
+          msg = "Error running ismaster on #{server.address}"
+          Utils.warn_monitor_exception(msg, exc,
+            logger: options[:logger],
+            log_prefix: options[:log_prefix],
+            bg_error_backtrace: options[:bg_error_backtrace],
+          )
+          {}
         end
-        result
       end
 
       def ismaster
@@ -274,6 +288,24 @@ module Mongo
             connection.handshake!
           end
           @connection = connection
+          if result['topologyVersion']
+            # Successful response, server 4.4+
+            @update_mutex.synchronize do
+              @push_monitor ||= PushMonitor.new(
+                self,
+                TopologyVersion.new(result['topologyVersion']),
+                monitoring,
+                **Utils.shallow_symbolize_keys(options.merge(
+                  socket_timeout: heartbeat_interval + connection.socket_timeout,
+                )),
+              )
+            end
+            push_monitor.run!
+          else
+            # Failed response or pre-4.4 server
+            stop_push_monitor!
+          end
+          result
         end
         result
       end