mongo 2.14.0 → 2.15.0.alpha

data/lib/mongo/auth/x509/conversation.rb

@@ -34,34 +34,13 @@ module Mongo
         # @param [ Server::Connection ] connection The connection being
         #   authenticated.
         #
-        # @return [ Protocol::Query ] The first X.509 conversation message.
+        # @return [ Protocol::Message ] The first X.509 conversation message.
         #
         # @since 2.0.0
         def start(connection)
-          login = client_first_document
-          if connection && connection.features.op_msg_enabled?
-            selector = login
-            # The only valid database for X.509 authentication is $external.
-            if user.auth_source != '$external'
-              user_name_msg = if user.name
-                " #{user.name}"
-              else
-                ''
-              end
-              raise Auth::InvalidConfiguration, "User#{user_name_msg} specifies auth source '#{user.auth_source}', but the only valid auth source for X.509 is '$external'"
-            end
-            selector[Protocol::Msg::DATABASE_IDENTIFIER] = '$external'
-            cluster_time = connection.mongos? && connection.cluster_time
-            selector[Operation::CLUSTER_TIME] = cluster_time if cluster_time
-            Protocol::Msg.new([], {}, selector)
-          else
-            Protocol::Query.new(
-              Auth::EXTERNAL,
-              Database::COMMAND,
-              login,
-              limit: -1
-            )
-          end
+          validate_external_auth_source
+          selector = client_first_document
+          build_message(connection, '$external', selector)
         end
 
         # Returns the hash to provide to the server in the handshake

data/lib/mongo/background_thread.rb

@@ -20,6 +20,17 @@ module Mongo
   # compatibility reasons. However using these methods outside of the driver
   # is deprecated.
   #
+  # @note Do not start or stop background threads in finalizers. See
+  #   https://jira.mongodb.org/browse/RUBY-2453 and
+  #   https://bugs.ruby-lang.org/issues/16288. When interpreter exits,
+  #   background threads are stopped first and finalizers are invoked next,
+  #   and MRI's internal data structures are basically corrupt at this point
+  #   if threads are being referenced. Prior to interpreter shutdown this
+  #   means threads cannot be stopped by objects going out of scope, but
+  #   most likely the threads hold references to said objects anyway if
+  #   work is being performed thus the objects wouldn't go out of scope in
+  #   the first place.
+  #
   # @api private
   module BackgroundThread
     include Loggable

data/lib/mongo/bulk_write.rb

@@ -58,6 +58,7 @@ module Mongo
       operations = op_combiner.combine
 
       client.send(:with_session, @options) do |session|
+        context = Operation::Context.new(client: client, session: session)
         operations.each do |operation|
           if single_statement?(operation)
             write_concern = write_concern(session)
@@ -67,6 +68,7 @@ module Mongo
                   operation.keys.first,
                   operation.values.flatten,
                   connection,
+                  context,
                   operation_id,
                   result_combiner,
                   session,
@@ -80,6 +82,7 @@ module Mongo
                   operation.keys.first,
                   operation.values.flatten,
                   connection,
+                  context,
                   operation_id,
                   result_combiner,
                   session)
@@ -177,19 +180,19 @@ module Mongo
       }
     end
 
-    def execute_operation(name, values, connection, operation_id, result_combiner, session, txn_num = nil)
+    def execute_operation(name, values, connection, context, operation_id, result_combiner, session, txn_num = nil)
       validate_collation!(connection)
       validate_array_filters!(connection)
       validate_hint!(connection)
 
       unpin_maybe(session) do
         if values.size > connection.description.max_write_batch_size
-          split_execute(name, values, connection, operation_id, result_combiner, session, txn_num)
+          split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
         else
-          result = send(name, values, connection, operation_id, session, txn_num)
+          result = send(name, values, connection, context, operation_id, session, txn_num)
 
           add_server_diagnostics(connection) do
-            add_error_labels(client, connection, session) do
+            add_error_labels(connection, context) do
               result_combiner.combine!(result, values.size)
             end
           end
@@ -205,7 +208,7 @@ module Mongo
     rescue Error::MaxBSONSize, Error::MaxMessageSize => e
       raise e if values.size <= 1
       unpin_maybe(session) do
-        split_execute(name, values, connection, operation_id, result_combiner, session, txn_num)
+        split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
       end
     end
 
@@ -213,47 +216,47 @@ module Mongo
       @op_combiner ||= ordered? ? OrderedCombiner.new(requests) : UnorderedCombiner.new(requests)
     end
 
-    def split_execute(name, values, connection, operation_id, result_combiner, session, txn_num)
-      execute_operation(name, values.shift(values.size / 2), connection, operation_id, result_combiner, session, txn_num)
+    def split_execute(name, values, connection, context, operation_id, result_combiner, session, txn_num)
+      execute_operation(name, values.shift(values.size / 2), connection, context, operation_id, result_combiner, session, txn_num)
 
       txn_num = session.next_txn_num if txn_num
-      execute_operation(name, values, connection, operation_id, result_combiner, session, txn_num)
+      execute_operation(name, values, connection, context, operation_id, result_combiner, session, txn_num)
     end
 
-    def delete_one(documents, connection, operation_id, session, txn_num)
+    def delete_one(documents, connection, context, operation_id, session, txn_num)
       QueryCache.clear_namespace(collection.namespace)
 
       spec = base_spec(operation_id, session).merge(:deletes => documents, :txn_num => txn_num)
-      Operation::Delete.new(spec).bulk_execute(connection, client: client)
+      Operation::Delete.new(spec).bulk_execute(connection, context: context)
     end
 
-    def delete_many(documents, connection, operation_id, session, txn_num)
+    def delete_many(documents, connection, context, operation_id, session, txn_num)
       QueryCache.clear_namespace(collection.namespace)
 
       spec = base_spec(operation_id, session).merge(:deletes => documents)
-      Operation::Delete.new(spec).bulk_execute(connection, client: client)
+      Operation::Delete.new(spec).bulk_execute(connection, context: context)
     end
 
-    def insert_one(documents, connection, operation_id, session, txn_num)
+    def insert_one(documents, connection, context, operation_id, session, txn_num)
       QueryCache.clear_namespace(collection.namespace)
 
       spec = base_spec(operation_id, session).merge(:documents => documents, :txn_num => txn_num)
-      Operation::Insert.new(spec).bulk_execute(connection, client: client)
+      Operation::Insert.new(spec).bulk_execute(connection, context: context)
     end
 
-    def update_one(documents, connection, operation_id, session, txn_num)
+    def update_one(documents, connection, context, operation_id, session, txn_num)
       QueryCache.clear_namespace(collection.namespace)
 
       spec = base_spec(operation_id, session).merge(:updates => documents, :txn_num => txn_num)
-      Operation::Update.new(spec).bulk_execute(connection, client: client)
+      Operation::Update.new(spec).bulk_execute(connection, context: context)
     end
     alias :replace_one :update_one
 
-    def update_many(documents, connection, operation_id, session, txn_num)
+    def update_many(documents, connection, context, operation_id, session, txn_num)
       QueryCache.clear_namespace(collection.namespace)
 
       spec = base_spec(operation_id, session).merge(:updates => documents)
-      Operation::Update.new(spec).bulk_execute(connection, client: client)
+      Operation::Update.new(spec).bulk_execute(connection, context: context)
     end
 
     private

data/lib/mongo/client.rb

@@ -85,6 +85,7 @@ module Mongo
       :retry_writes,
       :scan,
       :sdam_proc,
+      :server_api,
       :server_selection_timeout,
       :socket_timeout,
       :ssl,
@@ -114,7 +115,16 @@ module Mongo
     # The compression algorithms supported by the driver.
     #
     # @since 2.5.0
-    VALID_COMPRESSORS = [ Mongo::Protocol::Compressed::ZLIB ].freeze
+    VALID_COMPRESSORS = [
+      Mongo::Protocol::Compressed::ZSTD,
+      Mongo::Protocol::Compressed::SNAPPY,
+      Mongo::Protocol::Compressed::ZLIB
+    ].freeze
+
+    # The known server API versions.
+    VALID_SERVER_API_VERSIONS = %w(
+      1
+    ).freeze
 
     # @return [ Mongo::Cluster ] cluster The cluster of servers for the client.
     attr_reader :cluster
@@ -223,7 +233,7 @@ module Mongo
     # @option options [ Array<String> ] :compressors A list of potential
     #   compressors to use, in order of preference. The driver chooses the
     #   first compressor that is also supported by the server. Currently the
-    #   driver only supports 'zlib'.
+    #   driver only supports 'zstd, 'snappy' and 'zlib'.
     # @option options [ true | false ] :direct_connection Whether to connect
     #   directly to the specified seed, bypassing topology discovery. Exactly
     #   one seed must be provided.
@@ -312,6 +322,11 @@ module Mongo
     #   in particular the cluster is nil at this time. sdam_proc should
     #   limit itself to calling #subscribe and #unsubscribe methods on the
     #   client only.
+    # @option options [ Hash ] :server_api The requested server API version.
+    #   This hash can have the following items:
+    #   - *:version* -- string
+    #   - *:strict* -- boolean
+    #   - *:deprecation_errors* -- boolean
     # @option options [ Integer ] :server_selection_timeout The timeout in seconds
     #   for selecting a server for an operation.
     # @option options [ Float ] :socket_timeout The timeout, in seconds, to
@@ -465,6 +480,18 @@ module Mongo
 
       options = self.class.canonicalize_ruby_options(options)
 
+      # The server API version is specified to be a string.
+      # However, it is very annoying to always provide the number 1 as a string,
+      # therefore cast to the string type here.
+      if server_api = options[:server_api]
+        if server_api.is_a?(Hash)
+          server_api = Options::Redacted.new(server_api)
+          if (version = server_api[:version]).is_a?(Integer)
+            options[:server_api] = server_api.merge(version: version.to_s)
+          end
+        end
+      end
+
       # Special handling for sdam_proc as it is only used during client
       # construction
       sdam_proc = options.delete(:sdam_proc)
@@ -499,7 +526,9 @@ module Mongo
       validate_options!(addresses)
       validate_authentication_options!
 
-      @database = Database.new(self, @options[:database], @options)
+      database_options = @options.dup
+      database_options.delete(:server_api)
+      @database = Database.new(self, @options[:database], database_options)
 
       # Temporarily set monitoring so that event subscriptions can be
       # set up without there being a cluster
@@ -697,9 +726,9 @@ module Mongo
     # @return [ Mongo::Client ] A new client instance.
     #
     # @since 2.0.0
-    def with(new_options = Options::Redacted.new)
+    def with(new_options = nil)
       clone.tap do |client|
-        opts = client.update_options(new_options)
+        opts = client.update_options(new_options || Options::Redacted.new)
         Database.create(client)
         # We can't use the same cluster if some options that would affect it
         # have changed.
@@ -726,6 +755,8 @@ module Mongo
     def update_options(new_options)
       old_options = @options
 
+      new_options = self.class.canonicalize_ruby_options(new_options || {})
+
       validate_new_options!(new_options).tap do |opts|
         # Our options are frozen
         options = @options.dup
@@ -1127,7 +1158,7 @@ module Mongo
     # The argument may contain a subset of options that the client will
     # eventually have; this method validates each of the provided options
     # but does not check for interactions between combinations of options.
-    def validate_new_options!(opts = Options::Redacted.new)
+    def validate_new_options!(opts)
       return Options::Redacted.new unless opts
       if opts[:read_concern]
         # Raise an error for non user-settable options
@@ -1146,6 +1177,23 @@ module Mongo
         end
       end
 
+      if server_api = opts[:server_api]
+        unless server_api.is_a?(Hash)
+          raise ArgumentError, ":server_api value must be a hash: #{server_api}"
+        end
+
+        extra_keys = server_api.keys - %w(version strict deprecation_errors)
+        unless extra_keys.empty?
+          raise ArgumentError, "Unknown keys under :server_api: #{extra_keys.map(&:inspect).join(', ')}"
+        end
+
+        if version = server_api[:version]
+          unless VALID_SERVER_API_VERSIONS.include?(version)
+            raise ArgumentError, "Unknown server API version: #{version}"
+          end
+        end
+      end
+
       Lint.validate_underscore_read_preference(opts[:read])
       Lint.validate_read_concern_option(opts[:read_concern])
       opts.each.inject(Options::Redacted.new) do |_options, (k, v)|
@@ -1155,6 +1203,15 @@ module Mongo
           validate_read!(key, opts)
           if key == :compressors
             compressors = valid_compressors(v)
+
+            if compressors.include?('snappy')
+              validate_snappy_compression!
+            end
+
+            if compressors.include?('zstd')
+              validate_zstd_compression!
+            end
+
             _options[key] = compressors unless compressors.empty?
           else
             _options[key] = v
@@ -1311,11 +1368,30 @@ module Mongo
                        "This compressor will not be used.")
           false
         else
+
           true
         end
       end
     end
 
+    def validate_snappy_compression!
+      return if defined?(Snappy)
+      require 'snappy'
+    rescue LoadError => e
+      raise Error::UnmetDependency, "Cannot enable snappy compression because the snappy gem " \
+        "has not been installed. Add \"gem 'snappy'\" to your Gemfile and run " \
+        "\"bundle install\" to install the gem. (#{e.class}: #{e})"
+    end
+
+    def validate_zstd_compression!
+      return if defined?(Zstd)
+      require 'zstd-ruby'
+    rescue LoadError => e
+      raise Error::UnmetDependency, "Cannot enable zstd compression because the zstd-ruby gem " \
+        "has not been installed. Add \"gem 'zstd-ruby'\" to your Gemfile and run " \
+        "\"bundle install\" to install the gem. (#{e.class}: #{e})"
+    end
+
     def validate_max_min_pool_size!(option, opts)
       if option == :min_pool_size && opts[:min_pool_size]
         max = opts[:max_pool_size] || Server::ConnectionPool::DEFAULT_MAX_SIZE

data/lib/mongo/cluster.rb

@@ -107,6 +107,11 @@ module Mongo
     #   for the server monitor to refresh its description via ismaster.
     # @option options [ Hash ] :resolv_options For internal driver use only.
     #   Options to pass through to Resolv::DNS constructor for SRV lookups.
+    # @option options [ Hash ] :server_api The requested server API version.
+    #   This hash can have the following items:
+    #   - *:version* -- string
+    #   - *:strict* -- boolean
+    #   - *:deprecation_errors* -- boolean
     #
     # @since 2.0.0
     def initialize(seeds, monitoring, options = Options::Redacted.new)
@@ -129,7 +134,9 @@ module Mongo
       @servers = []
       @monitoring = monitoring
       @event_listeners = Event::Listeners.new
-      @app_metadata = Server::AppMetadata.new(@options)
+      @app_metadata = Server::AppMetadata.new(@options.merge(purpose: :application))
+      @monitor_app_metadata = Server::Monitor::AppMetadata.new(@options.merge(purpose: :monitor))
+      @push_monitor_app_metadata = Server::Monitor::AppMetadata.new(@options.merge(purpose: :push_monitor))
       @cluster_time_lock = Mutex.new
       @cluster_time = nil
       @srv_monitor_lock = Mutex.new
@@ -199,9 +206,6 @@ module Mongo
           @cursor_reaper, @socket_reaper,
         ], options)
 
-        ObjectSpace.define_finalizer(self, self.class.finalize(
-          {}, @periodic_executor, @session_pool))
-
         @periodic_executor.run!
       end
 
@@ -244,7 +248,11 @@ module Mongo
           # the servers list above and the wait call below, we should not
           # wait for the full remaining time - wait for up to 1 second, then
           # recheck the state.
-          server_selection_semaphore.wait([time_remaining, 1].min)
+          begin
+            server_selection_semaphore.wait([time_remaining, 1].min)
+          rescue ::Timeout::Error
+            # nothing
+          end
         end
       end
 
@@ -288,6 +296,12 @@ module Mongo
     # @since 2.4.0
     attr_reader :app_metadata
 
+    # @api private
+    attr_reader :monitor_app_metadata
+
+    # @api private
+    attr_reader :push_monitor_app_metadata
+
     # @return [ Array<String> ] The addresses of seed servers. Contains
     #   addresses that were given to Cluster when it was instantiated, not
     #   current addresses that the cluster is using as a result of SDAM.
@@ -445,25 +459,6 @@ module Mongo
     # @api private
     attr_reader :server_selection_semaphore
 
-    # Finalize the cluster for garbage collection.
-    #
-    # @example Finalize the cluster.
-    #   Cluster.finalize(pools)
-    #
-    # @param [ Hash<Address, Server::ConnectionPool> ] pools Ignored.
-    # @param [ PeriodicExecutor ] periodic_executor The periodic executor.
-    # @param [ SessionPool ] session_pool The session pool.
-    #
-    # @return [ Proc ] The Finalizer.
-    #
-    # @since 2.2.0
-    def self.finalize(pools, periodic_executor, session_pool)
-      proc do
-        session_pool.end_sessions
-        periodic_executor.stop!
-      end
-    end
-
     # Closes the cluster.
     #
     # @note Applications should call Client#close to disconnect from
@@ -953,10 +948,6 @@ module Mongo
             monitor_options = Utils.shallow_symbolize_keys(options.merge(
               timeout: options[:connect_timeout] || Server::CONNECT_TIMEOUT))
             @srv_monitor = _srv_monitor = Srv::Monitor.new(self, **monitor_options)
-            finalizer = lambda do
-              _srv_monitor.stop!
-            end
-            ObjectSpace.define_finalizer(self, finalizer)
           end
           @srv_monitor.run!
         end

data/lib/mongo/cluster/reapers/cursor_reaper.rb

@@ -130,16 +130,20 @@ module Mongo
         end
 
         to_kill_copy.each do |server, op_specs|
+          options = {
+            server_api: server.options[:server_api],
+          }
+          context = Operation::Context.new(options: options)
           op_specs.each do |op_spec|
             if server.features.find_command_enabled?
               Cursor::Builder::KillCursorsCommand.update_cursors(op_spec, active_cursors_copy.to_a)
               if Cursor::Builder::KillCursorsCommand.get_cursors_list(op_spec).size > 0
-                Operation::KillCursors.new(op_spec).execute(server, client: nil)
+                Operation::KillCursors.new(op_spec).execute(server, context: context)
               end
             else
               Cursor::Builder::OpKillCursors.update_cursors(op_spec, active_cursors_copy.to_a)
               if Cursor::Builder::OpKillCursors.get_cursors_list(op_spec).size > 0
-                Operation::KillCursors.new(op_spec).execute(server, client: nil)
+                Operation::KillCursors.new(op_spec).execute(server, context: context)
               end
             end
           end