mongo 2.14.0 → 2.18.1

data/lib/mongo/client.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
@@ -64,6 +67,7 @@ module Mongo
       :database,
       :heartbeat_frequency,
       :id_generator,
+      :load_balanced,
       :local_threshold,
       :logger,
       :log_prefix,
@@ -85,8 +89,11 @@ module Mongo
       :retry_writes,
       :scan,
       :sdam_proc,
+      :server_api,
       :server_selection_timeout,
       :socket_timeout,
+      :srv_max_hosts,
+      :srv_service_name,
       :ssl,
       :ssl_ca_cert,
       :ssl_ca_cert_object,
@@ -114,7 +121,16 @@ module Mongo
     # The compression algorithms supported by the driver.
     #
     # @since 2.5.0
-    VALID_COMPRESSORS = [ Mongo::Protocol::Compressed::ZLIB ].freeze
+    VALID_COMPRESSORS = [
+      Mongo::Protocol::Compressed::ZSTD,
+      Mongo::Protocol::Compressed::SNAPPY,
+      Mongo::Protocol::Compressed::ZLIB
+    ].freeze
+
+    # The known server API versions.
+    VALID_SERVER_API_VERSIONS = %w(
+      1
+    ).freeze
 
     # @return [ Mongo::Cluster ] cluster The cluster of servers for the client.
     attr_reader :cluster
@@ -223,21 +239,26 @@ module Mongo
     # @option options [ Array<String> ] :compressors A list of potential
     #   compressors to use, in order of preference. The driver chooses the
     #   first compressor that is also supported by the server. Currently the
-    #   driver only supports 'zlib'.
+    #   driver only supports 'zstd, 'snappy' and 'zlib'.
     # @option options [ true | false ] :direct_connection Whether to connect
     #   directly to the specified seed, bypassing topology discovery. Exactly
     #   one seed must be provided.
     # @option options [ Symbol ] :connect Deprecated - use :direct_connection
     #   option instead of this option. The connection method to use. This
     #   forces the cluster to behave in the specified way instead of
-    #   auto-discovering. One of :direct, :replica_set, :sharded
+    #   auto-discovering. One of :direct, :replica_set, :sharded,
+    #   :load_balanced. If :connect is set to :load_balanced, the driver
+    #   will behave as if the server is a load balancer even if it isn't
+    #   connected to a load balancer.
     # @option options [ Float ] :connect_timeout The timeout, in seconds, to
     #   attempt a connection.
     # @option options [ String ] :database The database to connect to.
     # @option options [ Float ] :heartbeat_frequency The interval, in seconds,
-    #   for the server monitor to refresh its description via ismaster.
+    #   for the server monitor to refresh its description via hello.
     # @option options [ Object ] :id_generator A custom object to generate ids
     #   for documents. Must respond to #generate.
+    # @option options [ true | false ] :load_balanced Whether to expect to
+    #   connect to a load balancer.
     # @option options [ Integer ] :local_threshold The local threshold boundary
     #   in seconds for selecting a near server for an operation.
     # @option options [ Logger ] :logger A custom logger to use.
@@ -247,7 +268,7 @@ module Mongo
     # @option options [ Integer ] :max_idle_time The maximum seconds a socket can remain idle
     #   since it has been checked in to the pool.
     # @option options [ Integer ] :max_pool_size The maximum size of the
-    #   connection pool.
+    #   connection pool. Setting this option to zero creates an unlimited connection pool.
     # @option options [ Integer ] :max_read_retries The maximum number of read
     #   retries when legacy read retries are in use.
     # @option options [ Integer ] :max_write_retries The maximum number of write
@@ -312,10 +333,22 @@ module Mongo
     #   in particular the cluster is nil at this time. sdam_proc should
     #   limit itself to calling #subscribe and #unsubscribe methods on the
     #   client only.
+    # @option options [ Hash ] :server_api The requested server API version.
+    #   This hash can have the following items:
+    #   - *:version* -- string
+    #   - *:strict* -- boolean
+    #   - *:deprecation_errors* -- boolean
     # @option options [ Integer ] :server_selection_timeout The timeout in seconds
     #   for selecting a server for an operation.
     # @option options [ Float ] :socket_timeout The timeout, in seconds, to
     #   execute operations on a socket.
+    # @option options [ Integer ] :srv_max_hosts The maximum number of mongoses
+    #   that the driver will communicate with for sharded topologies. If this
+    #   option is 0, then there will be no maximum number of mongoses. If the
+    #   given URI resolves to more hosts than ``:srv_max_hosts``, the client
+    #   will ramdomly choose an ``:srv_max_hosts`` sized subset of hosts.
+    # @option options [ String ] :srv_service_name The service name to use in
+    #   the SRV DNS query.
     # @option options [ true, false ] :ssl Whether to use TLS.
     # @option options [ String ] :ssl_ca_cert The file containing concatenated
     #   certificate authority certificates used to validate certs passed from the
@@ -396,11 +429,16 @@ module Mongo
     #     instance containing the encryption key vault
     #   - :key_vault_namespace => String, the namespace of the key vault in the
     #     format database.collection
-    #   - :kms_providers => Hash, A hash of key management service configuration
-    #     information. Valid hash keys are :local or :aws. There may be more
-    #     than one kms provider specified.
+    #   - :kms_providers => Hash, A hash of key management service (KMS) configuration
+    #     information. Valid hash keys are :aws, :azure, :gcp, :kmip, :local.
+    #     There may be more than one kms provider specified.
+    #   - :kms_tls_options => Hash, A hash of TLS options to authenticate to
+    #     KMS providers, usually used for KMIP servers. Valid hash keys
+    #     are :aws, :azure, :gcp, :kmip, :local. There may be more than one
+    #     kms provider specified.
     #   - :schema_map => Hash | nil, JSONSchema for one or more collections
-    #     specifying which fields should be encrypted.
+    #     specifying which fields should be encrypted. This option is
+    #     mutually exclusive with :schema_map_path.
     #     - Note: Schemas supplied in the schema_map only apply to configuring
     #       automatic encryption for client side encryption. Other validation
     #       rules in the JSON schema will not be enforced by the driver and will
@@ -409,10 +447,21 @@ module Mongo
     #       JSON Schemas obtained from the server. It protects against a
     #       malicious server advertising a false JSON Schema, which could trick
     #       the client into sending unencrypted data that should be encrypted.
+    #     - Note: If a collection is present on both the :encrypted_fields_map
+    #       and :schema_map, an error will be raised.
+    #   - :schema_map_path => String | nil A path to a file contains the JSON schema
+    #   of the collection that stores auto encrypted documents. This option is
+    #   mutually exclusive with :schema_map.
     #   - :bypass_auto_encryption => Boolean, when true, disables auto encryption;
     #     defaults to false.
     #   - :extra_options => Hash | nil, options related to spawning mongocryptd
     #     (this part of the API is subject to change).
+    #   - :encrypted_fields_map => Hash | nil, maps a collection namespace to
+    #     a hash describing encrypted fields for queryable encryption.
+    #     - Note: If a collection is present on both the encryptedFieldsMap
+    #       and schemaMap, an error will be raised.
+    #   - :bypass_query_analysis => Boolean | nil, when true disables automatic
+    #     analysis of outgoing commands.
     #
     #   Notes on automatic encryption:
     #   - Automatic encryption is an enterprise only feature that only applies
@@ -465,6 +514,18 @@ module Mongo
 
       options = self.class.canonicalize_ruby_options(options)
 
+      # The server API version is specified to be a string.
+      # However, it is very annoying to always provide the number 1 as a string,
+      # therefore cast to the string type here.
+      if server_api = options[:server_api]
+        if server_api.is_a?(Hash)
+          server_api = Options::Redacted.new(server_api)
+          if (version = server_api[:version]).is_a?(Integer)
+            options[:server_api] = server_api.merge(version: version.to_s)
+          end
+        end
+      end
+
       # Special handling for sdam_proc as it is only used during client
       # construction
       sdam_proc = options.delete(:sdam_proc)
@@ -496,10 +557,12 @@ module Mongo
       end
 =end
       @options.freeze
-      validate_options!(addresses)
+      validate_options!(addresses, is_srv: uri.is_a?(URI::SRVProtocol))
       validate_authentication_options!
 
-      @database = Database.new(self, @options[:database], @options)
+      database_options = @options.dup
+      database_options.delete(:server_api)
+      @database = Database.new(self, @options[:database], database_options)
 
       # Temporarily set monitoring so that event subscriptions can be
       # set up without there being a cluster
@@ -697,14 +760,14 @@ module Mongo
     # @return [ Mongo::Client ] A new client instance.
     #
     # @since 2.0.0
-    def with(new_options = Options::Redacted.new)
+    def with(new_options = nil)
       clone.tap do |client|
-        opts = client.update_options(new_options)
+        opts = client.update_options(new_options || Options::Redacted.new)
         Database.create(client)
         # We can't use the same cluster if some options that would affect it
         # have changed.
         if cluster_modifying?(opts)
-          Cluster.create(client)
+          Cluster.create(client, monitoring: opts[:monitoring])
         end
       end
     end
@@ -726,6 +789,8 @@ module Mongo
     def update_options(new_options)
       old_options = @options
 
+      new_options = self.class.canonicalize_ruby_options(new_options || {})
+
       validate_new_options!(new_options).tap do |opts|
         # Our options are frozen
         options = @options.dup
@@ -845,8 +910,11 @@ module Mongo
     #   which databases are returned based on user privileges when access control
     #   is enabled
     #
-    #   See https://docs.mongodb.com/manual/reference/command/listDatabases/
+    #   See https://mongodb.com/docs/manual/reference/command/listDatabases/
     #   for more information and usage.
+    # @option opts [ Session ] :session The session to use.
+    # @option options [ Object ] :comment A user-provided
+    #   comment to attach to this command.
     #
     # @return [ Array<String> ] The names of the databases.
     #
@@ -868,8 +936,11 @@ module Mongo
     #   which databases are returned based on user privileges when access control
     #   is enabled
     #
-    #   See https://docs.mongodb.com/manual/reference/command/listDatabases/
+    #   See https://mongodb.com/docs/manual/reference/command/listDatabases/
     #   for more information and usage.
+    # @option opts [ Session ] :session The session to use.
+    # @option options [ Object ] :comment A user-provided
+    #   comment to attach to this command.
     #
     # @return [ Array<Hash> ] The info for each database.
     #
@@ -890,6 +961,10 @@ module Mongo
     # @param [ Hash ] filter The filter criteria for getting a list of databases.
     # @param [ Hash ] opts The command options.
     #
+    # @option opts [ Session ] :session The session to use.
+    # @option options [ Object ] :comment A user-provided
+    #   comment to attach to this command.
+    #
     # @return [ Array<Mongo::Database> ] The list of database objects.
     #
     # @since 2.5.0
@@ -940,11 +1015,35 @@ module Mongo
     #
     # @param [ Array<Hash> ] pipeline Optional additional filter operators.
     # @param [ Hash ] options The change stream options.
+    # @option options [ String ] :full_document Allowed values: nil, 'default',
+    #   'updateLookup', 'whenAvailable', 'required'.
     #
-    # @option options [ String ] :full_document Allowed values: 'default', 'updateLookup'.
-    #   Defaults to 'default'. When set to 'updateLookup', the change notification for partial
-    #   updates will include both a delta describing the changes to the document, as well as a copy
-    #   of the entire document that was changed from some time after the change occurred.
+    #   The default is to not send a value (i.e. nil), which is equivalent to
+    #   'default'. By default, the change notification for partial updates will
+    #   include a delta describing the changes to the document.
+    #
+    #   When set to 'updateLookup', the change notification for partial updates
+    #   will include both a delta describing the changes to the document as well
+    #   as a copy of the entire document that was changed from some time after
+    #   the change occurred.
+    #
+    #   When set to 'whenAvailable', configures the change stream to return the
+    #   post-image of the modified document for replace and update change events
+    #   if the post-image for this event is available.
+    #
+    #   When set to 'required', the same behavior as 'whenAvailable' except that
+    #   an error is raised if the post-image is not available.
+    # @option options [ String ] :full_document_before_change Allowed values: nil,
+    #   'whenAvailable', 'required', 'off'.
+    #
+    #   The default is to not send a value (i.e. nil), which is equivalent to 'off'.
+    #
+    #   When set to 'whenAvailable', configures the change stream to return the
+    #   pre-image of the modified document for replace, update, and delete change
+    #   events if it is available.
+    #
+    #   When set to 'required', the same behavior as 'whenAvailable' except that
+    #   an error is raised if the pre-image is not available.
     # @option options [ BSON::Document, Hash ] :resume_after Specifies the logical starting point
     #   for the new change stream.
     # @option options [ Integer ] :max_await_time_ms The maximum amount of time for the server to
@@ -956,6 +1055,13 @@ module Mongo
     #   changes that occurred at or after the specified timestamp. Any command run
     #   against the server will return a cluster time that can be used here.
     #   Only recognized by server versions 4.0+.
+    # @option options [ Object ] :comment A user-provided
+    #   comment to attach to this command.
+    # @option options [ Boolean ] :show_expanded_events Enables the server to
+    #   send the 'expanded' list of change stream events. The list of additional
+    #   events included with this flag set are: createIndexes, dropIndexes,
+    #   modify, create, shardCollection, reshardCollection,
+    #   refineCollectionShardKey.
     #
     # @note A change stream only allows 'majority' read concern.
     # @note This helper method is preferable to running a raw aggregation with a $changeStream
@@ -967,8 +1073,11 @@ module Mongo
     def watch(pipeline = [], options = {})
       return use(Database::ADMIN).watch(pipeline, options) unless database.name == Database::ADMIN
 
+      view_options = options.dup
+      view_options[:await_data] = true if options[:max_await_time_ms]
+
       Mongo::Collection::View::ChangeStream.new(
-        Mongo::Collection::View.new(self["#{Database::COMMAND}.aggregate"]),
+        Mongo::Collection::View.new(self["#{Database::COMMAND}.aggregate"], {}, view_options),
         pipeline,
         Mongo::Collection::View::ChangeStream::CLUSTER,
         options)
@@ -1044,6 +1153,14 @@ module Mongo
       end
     end
 
+    # Returns encrypted field map hash if provided when creating the client.
+    #
+    # @return [ Hash | nil ] Encrypted field map hash, or nil if not set.
+    # @api private
+    def encrypted_fields_map
+      @encrypted_fields_map ||= @options.fetch(:auto_encryption_options, {})[:encrypted_fields_map]
+    end
+
     private
 
     # Create a new encrypter object using the client's auto encryption options
@@ -1103,11 +1220,26 @@ module Mongo
 
       cluster.validate_session_support!
 
-      Session.new(cluster.session_pool.checkout, self, { implicit: true }.merge(options))
+      options = {implicit: true}.update(options)
+
+      server_session = if options[:implicit]
+        nil
+      else
+        cluster.session_pool.checkout
+      end
+
+      Session.new(server_session, self, options)
     end
 
+    # Auxiliary method that is called by interpreter when copying the client
+    # via dup or clone.
+    #
+    # @param [ Mongo::Client ] original Client that is being cloned.
+    #
+    # @api private
     def initialize_copy(original)
       @options = original.options.dup
+      @connect_lock = Mutex.new
       @monitoring = @cluster ? monitoring : Monitoring.new(options)
       @database = nil
       @read_preference = nil
@@ -1127,7 +1259,7 @@ module Mongo
     # The argument may contain a subset of options that the client will
     # eventually have; this method validates each of the provided options
     # but does not check for interactions between combinations of options.
-    def validate_new_options!(opts = Options::Redacted.new)
+    def validate_new_options!(opts)
       return Options::Redacted.new unless opts
       if opts[:read_concern]
         # Raise an error for non user-settable options
@@ -1146,6 +1278,23 @@ module Mongo
         end
       end
 
+      if server_api = opts[:server_api]
+        unless server_api.is_a?(Hash)
+          raise ArgumentError, ":server_api value must be a hash: #{server_api}"
+        end
+
+        extra_keys = server_api.keys - %w(version strict deprecation_errors)
+        unless extra_keys.empty?
+          raise ArgumentError, "Unknown keys under :server_api: #{extra_keys.map(&:inspect).join(', ')}"
+        end
+
+        if version = server_api[:version]
+          unless VALID_SERVER_API_VERSIONS.include?(version)
+            raise ArgumentError, "Unknown server API version: #{version}"
+          end
+        end
+      end
+
       Lint.validate_underscore_read_preference(opts[:read])
       Lint.validate_read_concern_option(opts[:read_concern])
       opts.each.inject(Options::Redacted.new) do |_options, (k, v)|
@@ -1155,7 +1304,22 @@ module Mongo
           validate_read!(key, opts)
           if key == :compressors
             compressors = valid_compressors(v)
+
+            if compressors.include?('snappy')
+              validate_snappy_compression!
+            end
+
+            if compressors.include?('zstd')
+              validate_zstd_compression!
+            end
+
             _options[key] = compressors unless compressors.empty?
+          elsif key == :srv_max_hosts
+            if v && (!v.is_a?(Integer) || v < 0)
+              log_warn("#{v} is not a valid integer for srv_max_hosts")
+            else
+              _options[key] = v
+            end
           else
             _options[key] = v
           end
@@ -1169,14 +1333,20 @@ module Mongo
     # Validates all options after they are set on the client.
     # This method is intended to catch combinations of options which are
     # not allowed.
-    def validate_options!(addresses = nil)
+    def validate_options!(addresses = nil, is_srv: nil)
       if options[:write] && options[:write_concern] && options[:write] != options[:write_concern]
         raise ArgumentError, "If :write and :write_concern are both given, they must be identical: #{options.inspect}"
       end
 
+      connect = options[:connect]&.to_sym
+
+      if connect && !%i(direct replica_set sharded load_balanced).include?(connect)
+        raise ArgumentError, "Invalid :connect option value: #{connect}"
+      end
+
       if options[:direct_connection]
-        if options[:connect] && options[:connect].to_sym != :direct
-          raise ArgumentError, "Conflicting client options: direct_connection=true and connect=#{options[:connect]}"
+        if connect && connect != :direct
+          raise ArgumentError, "Conflicting client options: direct_connection=true and connect=#{connect}"
         end
         # When a new client is created, we get the list of seed addresses
         if addresses && addresses.length > 1
@@ -1188,8 +1358,36 @@ module Mongo
         end
       end
 
-      if options[:direct_connection] == false && options[:connect] && options[:connect].to_sym == :direct
-        raise ArgumentError, "Conflicting client options: direct_connection=false and connect=#{options[:connect]}"
+      if options[:load_balanced]
+        if addresses && addresses.length > 1
+          raise ArgumentError, "load_balanced=true cannot be used with multiple seeds"
+        end
+
+        if options[:direct_connection]
+          raise ArgumentError, "direct_connection=true cannot be used with load_balanced=true"
+        end
+
+        if connect && connect != :load_balanced
+          raise ArgumentError, "connect=#{connect} cannot be used with load_balanced=true"
+        end
+
+        if options[:replica_set]
+          raise ArgumentError, "load_balanced=true cannot be used with replica_set option"
+        end
+      end
+
+      if connect == :load_balanced
+        if addresses && addresses.length > 1
+          raise ArgumentError, "connect=load_balanced cannot be used with multiple seeds"
+        end
+
+        if options[:replica_set]
+          raise ArgumentError, "connect=load_balanced cannot be used with replica_set option"
+        end
+      end
+
+      if options[:direct_connection] == false && connect && connect == :direct
+        raise ArgumentError, "Conflicting client options: direct_connection=false and connect=#{connect}"
       end
 
       %i(connect_timeout socket_timeout).each do |key|
@@ -1245,6 +1443,26 @@ module Mongo
           end
         end
       end
+
+      if options[:srv_max_hosts] && options[:srv_max_hosts] > 0
+        if options[:replica_set]
+          raise ArgumentError, ":srv_max_hosts > 0 cannot be used with :replica_set option"
+        end
+
+        if options[:load_balanced]
+          raise ArgumentError, ":srv_max_hosts > 0 cannot be used with :load_balanced=true"
+        end
+      end
+
+      unless is_srv.nil? || is_srv
+        if options[:srv_max_hosts]
+          raise ArgumentError, ":srv_max_hosts cannot be used on non-SRV URI"
+        end
+
+        if options[:srv_service_name]
+          raise ArgumentError, ":srv_service_name cannot be used on non-SRV URI"
+        end
+      end
     end
 
     # Validates all authentication-related options after they are set on the client
@@ -1311,15 +1529,36 @@ module Mongo
                        "This compressor will not be used.")
           false
         else
+
           true
         end
       end
     end
 
+    def validate_snappy_compression!
+      return if defined?(Snappy)
+      require 'snappy'
+    rescue LoadError => e
+      raise Error::UnmetDependency, "Cannot enable snappy compression because the snappy gem " \
+        "has not been installed. Add \"gem 'snappy'\" to your Gemfile and run " \
+        "\"bundle install\" to install the gem. (#{e.class}: #{e})"
+    end
+
+    def validate_zstd_compression!
+      return if defined?(Zstd)
+      require 'zstd-ruby'
+    rescue LoadError => e
+      raise Error::UnmetDependency, "Cannot enable zstd compression because the zstd-ruby gem " \
+        "has not been installed. Add \"gem 'zstd-ruby'\" to your Gemfile and run " \
+        "\"bundle install\" to install the gem. (#{e.class}: #{e})"
+    end
+
     def validate_max_min_pool_size!(option, opts)
       if option == :min_pool_size && opts[:min_pool_size]
         max = opts[:max_pool_size] || Server::ConnectionPool::DEFAULT_MAX_SIZE
-        raise Error::InvalidMinPoolSize.new(opts[:min_pool_size], max) unless opts[:min_pool_size] <= max
+        if max != 0 && opts[:min_pool_size] > max
+          raise Error::InvalidMinPoolSize.new(opts[:min_pool_size], max)
+        end
       end
       true
     end