mongo 2.14.0 → 2.16.0

data/lib/mongo/client.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
@@ -64,6 +67,7 @@ module Mongo
       :database,
       :heartbeat_frequency,
       :id_generator,
+      :load_balanced,
       :local_threshold,
       :logger,
       :log_prefix,
@@ -85,6 +89,7 @@ module Mongo
       :retry_writes,
       :scan,
       :sdam_proc,
+      :server_api,
       :server_selection_timeout,
       :socket_timeout,
       :ssl,
@@ -114,7 +119,16 @@ module Mongo
     # The compression algorithms supported by the driver.
     #
     # @since 2.5.0
-    VALID_COMPRESSORS = [ Mongo::Protocol::Compressed::ZLIB ].freeze
+    VALID_COMPRESSORS = [
+      Mongo::Protocol::Compressed::ZSTD,
+      Mongo::Protocol::Compressed::SNAPPY,
+      Mongo::Protocol::Compressed::ZLIB
+    ].freeze
+
+    # The known server API versions.
+    VALID_SERVER_API_VERSIONS = %w(
+      1
+    ).freeze
 
     # @return [ Mongo::Cluster ] cluster The cluster of servers for the client.
     attr_reader :cluster
@@ -223,21 +237,26 @@ module Mongo
     # @option options [ Array<String> ] :compressors A list of potential
     #   compressors to use, in order of preference. The driver chooses the
     #   first compressor that is also supported by the server. Currently the
-    #   driver only supports 'zlib'.
+    #   driver only supports 'zstd, 'snappy' and 'zlib'.
     # @option options [ true | false ] :direct_connection Whether to connect
     #   directly to the specified seed, bypassing topology discovery. Exactly
     #   one seed must be provided.
     # @option options [ Symbol ] :connect Deprecated - use :direct_connection
     #   option instead of this option. The connection method to use. This
     #   forces the cluster to behave in the specified way instead of
-    #   auto-discovering. One of :direct, :replica_set, :sharded
+    #   auto-discovering. One of :direct, :replica_set, :sharded,
+    #   :load_balanced. If :connect is set to :load_balanced, the driver
+    #   will behave as if the server is a load balancer even if it isn't
+    #   connected to a load balancer.
     # @option options [ Float ] :connect_timeout The timeout, in seconds, to
     #   attempt a connection.
     # @option options [ String ] :database The database to connect to.
     # @option options [ Float ] :heartbeat_frequency The interval, in seconds,
-    #   for the server monitor to refresh its description via ismaster.
+    #   for the server monitor to refresh its description via hello.
     # @option options [ Object ] :id_generator A custom object to generate ids
     #   for documents. Must respond to #generate.
+    # @option options [ true | false ] :load_balanced Whether to expect to
+    #   connect to a load balancer.
     # @option options [ Integer ] :local_threshold The local threshold boundary
     #   in seconds for selecting a near server for an operation.
     # @option options [ Logger ] :logger A custom logger to use.
@@ -312,6 +331,11 @@ module Mongo
     #   in particular the cluster is nil at this time. sdam_proc should
     #   limit itself to calling #subscribe and #unsubscribe methods on the
     #   client only.
+    # @option options [ Hash ] :server_api The requested server API version.
+    #   This hash can have the following items:
+    #   - *:version* -- string
+    #   - *:strict* -- boolean
+    #   - *:deprecation_errors* -- boolean
     # @option options [ Integer ] :server_selection_timeout The timeout in seconds
     #   for selecting a server for an operation.
     # @option options [ Float ] :socket_timeout The timeout, in seconds, to
@@ -465,6 +489,18 @@ module Mongo
 
       options = self.class.canonicalize_ruby_options(options)
 
+      # The server API version is specified to be a string.
+      # However, it is very annoying to always provide the number 1 as a string,
+      # therefore cast to the string type here.
+      if server_api = options[:server_api]
+        if server_api.is_a?(Hash)
+          server_api = Options::Redacted.new(server_api)
+          if (version = server_api[:version]).is_a?(Integer)
+            options[:server_api] = server_api.merge(version: version.to_s)
+          end
+        end
+      end
+
       # Special handling for sdam_proc as it is only used during client
       # construction
       sdam_proc = options.delete(:sdam_proc)
@@ -499,7 +535,9 @@ module Mongo
       validate_options!(addresses)
       validate_authentication_options!
 
-      @database = Database.new(self, @options[:database], @options)
+      database_options = @options.dup
+      database_options.delete(:server_api)
+      @database = Database.new(self, @options[:database], database_options)
 
       # Temporarily set monitoring so that event subscriptions can be
       # set up without there being a cluster
@@ -697,9 +735,9 @@ module Mongo
     # @return [ Mongo::Client ] A new client instance.
     #
     # @since 2.0.0
-    def with(new_options = Options::Redacted.new)
+    def with(new_options = nil)
       clone.tap do |client|
-        opts = client.update_options(new_options)
+        opts = client.update_options(new_options || Options::Redacted.new)
         Database.create(client)
         # We can't use the same cluster if some options that would affect it
         # have changed.
@@ -726,6 +764,8 @@ module Mongo
     def update_options(new_options)
       old_options = @options
 
+      new_options = self.class.canonicalize_ruby_options(new_options || {})
+
       validate_new_options!(new_options).tap do |opts|
         # Our options are frozen
         options = @options.dup
@@ -1127,7 +1167,7 @@ module Mongo
     # The argument may contain a subset of options that the client will
     # eventually have; this method validates each of the provided options
     # but does not check for interactions between combinations of options.
-    def validate_new_options!(opts = Options::Redacted.new)
+    def validate_new_options!(opts)
       return Options::Redacted.new unless opts
       if opts[:read_concern]
         # Raise an error for non user-settable options
@@ -1146,6 +1186,23 @@ module Mongo
         end
       end
 
+      if server_api = opts[:server_api]
+        unless server_api.is_a?(Hash)
+          raise ArgumentError, ":server_api value must be a hash: #{server_api}"
+        end
+
+        extra_keys = server_api.keys - %w(version strict deprecation_errors)
+        unless extra_keys.empty?
+          raise ArgumentError, "Unknown keys under :server_api: #{extra_keys.map(&:inspect).join(', ')}"
+        end
+
+        if version = server_api[:version]
+          unless VALID_SERVER_API_VERSIONS.include?(version)
+            raise ArgumentError, "Unknown server API version: #{version}"
+          end
+        end
+      end
+
       Lint.validate_underscore_read_preference(opts[:read])
       Lint.validate_read_concern_option(opts[:read_concern])
       opts.each.inject(Options::Redacted.new) do |_options, (k, v)|
@@ -1155,6 +1212,15 @@ module Mongo
           validate_read!(key, opts)
           if key == :compressors
             compressors = valid_compressors(v)
+
+            if compressors.include?('snappy')
+              validate_snappy_compression!
+            end
+
+            if compressors.include?('zstd')
+              validate_zstd_compression!
+            end
+
             _options[key] = compressors unless compressors.empty?
           else
             _options[key] = v
@@ -1174,9 +1240,15 @@ module Mongo
         raise ArgumentError, "If :write and :write_concern are both given, they must be identical: #{options.inspect}"
       end
 
+      connect = options[:connect]&.to_sym
+
+      if connect && !%i(direct replica_set sharded load_balanced).include?(connect)
+        raise ArgumentError, "Invalid :connect option value: #{connect}"
+      end
+
       if options[:direct_connection]
-        if options[:connect] && options[:connect].to_sym != :direct
-          raise ArgumentError, "Conflicting client options: direct_connection=true and connect=#{options[:connect]}"
+        if connect && connect != :direct
+          raise ArgumentError, "Conflicting client options: direct_connection=true and connect=#{connect}"
         end
         # When a new client is created, we get the list of seed addresses
         if addresses && addresses.length > 1
@@ -1188,8 +1260,36 @@ module Mongo
         end
       end
 
-      if options[:direct_connection] == false && options[:connect] && options[:connect].to_sym == :direct
-        raise ArgumentError, "Conflicting client options: direct_connection=false and connect=#{options[:connect]}"
+      if options[:load_balanced]
+        if addresses && addresses.length > 1
+          raise ArgumentError, "load_balanced=true cannot be used with multiple seeds"
+        end
+
+        if options[:direct_connection]
+          raise ArgumentError, "direct_connection=true cannot be used with load_balanced=true"
+        end
+
+        if connect && connect != :load_balanced
+          raise ArgumentError, "connect=#{connect} cannot be used with load_balanced=true"
+        end
+
+        if options[:replica_set]
+          raise ArgumentError, "load_balanced=true cannot be used with replica_set option"
+        end
+      end
+
+      if connect == :load_balanced
+        if addresses && addresses.length > 1
+          raise ArgumentError, "connect=load_balanced cannot be used with multiple seeds"
+        end
+
+        if options[:replica_set]
+          raise ArgumentError, "connect=load_balanced cannot be used with replica_set option"
+        end
+      end
+
+      if options[:direct_connection] == false && connect && connect == :direct
+        raise ArgumentError, "Conflicting client options: direct_connection=false and connect=#{connect}"
       end
 
       %i(connect_timeout socket_timeout).each do |key|
@@ -1311,11 +1411,30 @@ module Mongo
                        "This compressor will not be used.")
           false
         else
+
           true
         end
       end
     end
 
+    def validate_snappy_compression!
+      return if defined?(Snappy)
+      require 'snappy'
+    rescue LoadError => e
+      raise Error::UnmetDependency, "Cannot enable snappy compression because the snappy gem " \
+        "has not been installed. Add \"gem 'snappy'\" to your Gemfile and run " \
+        "\"bundle install\" to install the gem. (#{e.class}: #{e})"
+    end
+
+    def validate_zstd_compression!
+      return if defined?(Zstd)
+      require 'zstd-ruby'
+    rescue LoadError => e
+      raise Error::UnmetDependency, "Cannot enable zstd compression because the zstd-ruby gem " \
+        "has not been installed. Add \"gem 'zstd-ruby'\" to your Gemfile and run " \
+        "\"bundle install\" to install the gem. (#{e.class}: #{e})"
+    end
+
     def validate_max_min_pool_size!(option, opts)
       if option == :min_pool_size && opts[:min_pool_size]
         max = opts[:max_pool_size] || Server::ConnectionPool::DEFAULT_MAX_SIZE

data/lib/mongo/client_encryption.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2019-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/cluster/periodic_executor.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,14 +36,15 @@ module Mongo
       #
       # @example Create a PeriodicExecutor.
       #   Mongo::Cluster::PeriodicExecutor.new([reaper, reaper2])
+      #
+      # @param [ Array<Object> ] executors The executors. Each must respond
+      #   to #execute and #flush.
       # @param [ Hash ] options The options.
       #
       # @option options [ Logger ] :logger A custom logger to use.
       #
       # @api private
-      #
-      # @since 2.5.0
-      def initialize(executors = [], options = {})
+      def initialize(executors, options = {})
         @thread = nil
         @executors = executors
         @stop_semaphore = Semaphore.new

data/lib/mongo/cluster/reapers/cursor_reaper.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,42 +28,38 @@ module Mongo
     class CursorReaper
       include Retryable
 
-      # The default time interval for the cursor reaper to send pending kill cursors operations.
+      # The default time interval for the cursor reaper to send pending
+      # kill cursors operations.
       #
       # @since 2.3.0
       FREQUENCY = 1.freeze
 
       # Create a cursor reaper.
       #
-      # @example Create a CursorReaper.
-      #   Mongo::Cluster::CursorReaper.new(cluster)
+      # @param [ Cluster ] cluster The cluster.
       #
       # @api private
-      #
-      # @since 2.3.0
-      def initialize
+      def initialize(cluster)
+        @cluster = cluster
         @to_kill = {}
-        @active_cursors = Set.new
+        @active_cursor_ids = Set.new
         @mutex = Mutex.new
       end
 
+      attr_reader :cluster
+
       # Schedule a kill cursors operation to be eventually executed.
       #
-      # @example Schedule a kill cursors operation.
-      #   cursor_reaper.schedule_kill_cursor(id, op_spec, server)
-      #
-      # @param [ Integer ] id The id of the cursor to kill.
-      # @param [ Hash ] op_spec The spec for the kill cursors op.
-      # @param [ Mongo::Server ] server The server to send the kill cursors operation to.
+      # @param [ Cursor::KillSpec ] kill_spec The kill specification.
+      # @param [ Mongo::Server ] server The server to send the kill cursors
+      #   operation to.
       #
       # @api private
-      #
-      # @since 2.3.0
-      def schedule_kill_cursor(id, op_spec, server)
+      def schedule_kill_cursor(kill_spec, server)
         @mutex.synchronize do
-          if @active_cursors.include?(id)
-            @to_kill[server] ||= Set.new
-            @to_kill[server] << op_spec
+          if @active_cursor_ids.include?(kill_spec.cursor_id)
+            @to_kill[server.address.seed] ||= Set.new
+            @to_kill[server.address.seed] << kill_spec
           end
         end
       end
@@ -84,7 +83,7 @@ module Mongo
         end
 
         @mutex.synchronize do
-          @active_cursors << id
+          @active_cursor_ids << id
         end
       end
 
@@ -107,7 +106,7 @@ module Mongo
         end
 
         @mutex.synchronize do
-          @active_cursors.delete(id)
+          @active_cursor_ids.delete(id)
         end
       end
 
@@ -120,29 +119,70 @@ module Mongo
       #
       # @since 2.3.0
       def kill_cursors
-        to_kill_copy = {}
-        active_cursors_copy = []
+        # TODO optimize this to batch kill cursor operations for the same
+        # server/database/collection instead of killing each cursor
+        # individually.
 
-        @mutex.synchronize do
-          to_kill_copy = @to_kill.dup
-          active_cursors_copy = @active_cursors.dup
-          @to_kill = {}
-        end
+        loop do
+          server_address_str = nil
 
-        to_kill_copy.each do |server, op_specs|
-          op_specs.each do |op_spec|
-            if server.features.find_command_enabled?
-              Cursor::Builder::KillCursorsCommand.update_cursors(op_spec, active_cursors_copy.to_a)
-              if Cursor::Builder::KillCursorsCommand.get_cursors_list(op_spec).size > 0
-                Operation::KillCursors.new(op_spec).execute(server, client: nil)
-              end
-            else
-              Cursor::Builder::OpKillCursors.update_cursors(op_spec, active_cursors_copy.to_a)
-              if Cursor::Builder::OpKillCursors.get_cursors_list(op_spec).size > 0
-                Operation::KillCursors.new(op_spec).execute(server, client: nil)
-              end
+          kill_spec = @mutex.synchronize do
+            # Find a server that has any cursors scheduled for destruction.
+            server_address_str, specs =
+              @to_kill.detect { |server_address_str, specs| specs.any? }
+
+            if specs.nil?
+              # All servers have empty specs, nothing to do.
+              return
             end
+
+            # Note that this mutates the spec in the queue.
+            # If the kill cursor operation fails, we don't attempt to
+            # kill that cursor again.
+            spec = specs.take(1).tap do |arr|
+              specs.subtract(arr)
+            end.first
+
+            unless @active_cursor_ids.include?(spec.cursor_id)
+              # The cursor was already killed, typically because it has
+              # been iterated to completion. Remove the kill spec from
+              # our records without doing any more work.
+              spec = nil
+            end
+
+            spec
+          end
+
+          # If there was a spec to kill but its cursor was already killed,
+          # look for another spec.
+          next unless kill_spec
+
+          # We could also pass kill_spec directly into the KillCursors
+          # operation, though this would make that operation have a
+          # different API from all of the other ones which accept hashes.
+          spec = {
+            cursor_ids: [kill_spec.cursor_id],
+            coll_name: kill_spec.coll_name,
+            db_name: kill_spec.db_name,
+          }
+          op = Operation::KillCursors.new(spec)
+
+          server = cluster.servers.detect do |server|
+            server.address.seed == server_address_str
           end
+
+          unless server
+            # TODO We currently don't have a server for the address that the
+            # cursor is associated with. We should leave the cursor in the
+            # queue to be killed at a later time (when the server comes back).
+            next
+          end
+
+          options = {
+            server_api: server.options[:server_api],
+            service_id: kill_spec.service_id,
+          }
+          op.execute(server, context: Operation::Context.new(options: options))
         end
       end
       alias :execute :kill_cursors

data/lib/mongo/cluster/reapers/socket_reaper.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/cluster/sdam_flow.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2018-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -102,6 +105,11 @@ class Mongo::Cluster
       end
 
       case topology
+      when Topology::LoadBalanced
+        @updated_desc = ::Mongo::Server::Description::LoadBalancer.new(
+          updated_desc.address,
+        )
+        update_server_descriptions
       when Topology::Single
         if topology.replica_set_name
           if updated_desc.replica_set_name != topology.replica_set_name
@@ -109,7 +117,7 @@ class Mongo::Cluster
               "Server #{updated_desc.address.to_s} has an incorrect replica set name '#{updated_desc.replica_set_name}'; expected '#{topology.replica_set_name}'"
             )
             @updated_desc = ::Mongo::Server::Description.new(updated_desc.address,
-              {}, updated_desc.average_round_trip_time)
+              {}, average_round_trip_time: updated_desc.average_round_trip_time)
             update_server_descriptions
           end
         end
@@ -178,6 +186,7 @@ class Mongo::Cluster
         raise ArgumentError, "Unknown topology #{topology.class}"
       end
 
+      verify_invariants
       commit_changes
       disconnect_servers
     end
@@ -225,7 +234,7 @@ class Mongo::Cluster
 
       if stale_primary?
         @updated_desc = ::Mongo::Server::Description.new(updated_desc.address,
-          {}, updated_desc.average_round_trip_time)
+          {}, average_round_trip_time: updated_desc.average_round_trip_time)
         update_server_descriptions
         check_if_has_primary
         return
@@ -254,7 +263,8 @@ class Mongo::Cluster
         if server.address != updated_desc.address
           if server.primary?
             server.update_description(::Mongo::Server::Description.new(
-              server.address, {}, server.description.average_round_trip_time))
+              server.address, {},
+              average_round_trip_time: server.description.average_round_trip_time))
           end
         end
       end
@@ -363,6 +373,9 @@ class Mongo::Cluster
           end
         end
       end
+
+      verify_invariants
+
       added_servers
     end
 
@@ -599,5 +612,15 @@ class Mongo::Cluster
 
       @previous_server_descriptions != server_descriptions
     end
+
+    def verify_invariants
+      if Mongo::Lint.enabled?
+        if cluster.topology.single?
+          if cluster.servers_list.length > 1
+            raise Mongo::Error::LintError, "Trying to create a single topology with multiple servers: #{cluster.servers_list}"
+          end
+        end
+      end
+    end
   end
 end

data/lib/mongo/cluster/topology/base.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2018-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -64,17 +67,21 @@ module Mongo
             @server_descriptions[server.address.to_s] = server.description
           end
 
-          begin
-            server_descriptions.each do |address_str, desc|
-              unless desc.unknown?
-                desc.features.check_driver_support!
+          if is_a?(LoadBalanced)
+            @compatible = true
+          else
+            begin
+              server_descriptions.each do |address_str, desc|
+                unless desc.unknown?
+                  desc.features.check_driver_support!
+                end
               end
+            rescue Error::UnsupportedFeatures => e
+              @compatible = false
+              @compatibility_error = e
+            else
+              @compatible = true
             end
-          rescue Error::UnsupportedFeatures => e
-            @compatible = false
-            @compatibility_error = e
-          else
-            @compatible = true
           end
 
           @have_data_bearing_servers = false