mongo 2.24.0 → 2.25.0

data/lib/mongo/collection/view/change_stream.rb

@@ -351,19 +351,24 @@ module Mongo
               # In load balanced topology, manually check out a connection
               # so it remains checked out and pinned to the cursor.
               connection = server.pool.check_out(context: context)
-              result = send_initial_query(connection, context)
-
-              start_at_operation_time = if doc = result.replies.first && result.replies.first.documents.first
-                                          doc['operationTime']
-                                        else
-                                          nil
-                                        end
-              result
+              begin
+                result = send_initial_query(connection, context)
+
+                start_at_operation_time = if (doc = result.replies.first && result.replies.first.documents.first)
+                                            doc['operationTime']
+                                          else
+                                            nil
+                                          end
+                result
+              rescue StandardError
+                server.pool.check_in(connection)
+                raise
+              end
             else
               server.with_connection do |connection|
                 result = send_initial_query(connection, context)
 
-                start_at_operation_time = if doc = result.replies.first && result.replies.first.documents.first
+                start_at_operation_time = if (doc = result.replies.first && result.replies.first.documents.first)
                                             doc['operationTime']
                                           else
                                             # The above may set @start_at_operation_time to nil

data/lib/mongo/collection/view/map_reduce.rb

@@ -227,7 +227,7 @@ module Mongo
         #
         # @since 2.5.0
         def execute
-          view.send(:with_session, @options) do |session|
+          view.with_session(@options) do |session|
             write_concern = view.write_concern_with_session(session)
             context = Operation::Context.new(client: client, session: session)
             nro_write_with_retry(write_concern, context: context) do |connection, _txn_num, context|
@@ -241,7 +241,7 @@ module Mongo
         OUT_ACTIONS = %i[replace merge reduce].freeze
 
         def server_selector
-          @view.send(:server_selector)
+          @view.server_selector
         end
 
         def inline?
@@ -299,15 +299,6 @@ module Mongo
           end
         end
 
-        def fetch_query_spec
-          Builder::MapReduce.new(map_function, reduce_function, view, options).query_specification
-        end
-
-        def find_command_spec(session)
-          Builder::MapReduce.new(map_function, reduce_function, view,
-                                 options.merge(session: session)).command_specification
-        end
-
         def fetch_query_op(session)
           spec = {
             coll_name: out_collection_name,

data/lib/mongo/collection/view/readable.rb

@@ -678,7 +678,7 @@ module Mongo
         # @api private
         def read_concern
           if options[:session] && options[:session].in_transaction?
-            options[:session].send(:txn_read_concern) || collection.client.read_concern
+            options[:session].txn_read_concern || collection.client.read_concern
           else
             collection.read_concern
           end
@@ -749,12 +749,10 @@ module Mongo
           end
         end
 
-        private
-
-        def collation(doc = nil)
-          configure(:collation, doc)
-        end
-
+        # The server selector for this view, derived from its read
+        # preference (or the collection/client default).
+        #
+        # @api private
         def server_selector
           @server_selector ||= if options[:session] && options[:session].in_transaction?
                                  ServerSelector.get(read_preference || client.server_selector)
@@ -763,6 +761,12 @@ module Mongo
                                end
         end
 
+        private
+
+        def collation(doc = nil)
+          configure(:collation, doc)
+        end
+
         def validate_doc!(doc)
           raise Error::InvalidDocument.new unless doc.respond_to?(:keys)
         end

data/lib/mongo/collection/view/writable.rb

@@ -624,8 +624,8 @@ module Mongo
         # @raise [ Error::InvalidUpdateDocument ] if the first key in the
         #   document does not start with a $.
         def validate_update_documents!(spec)
-          return unless update = spec.is_a?(Array) ? spec&.first : spec
-          return unless key = update.keys&.first
+          return unless (update = spec.is_a?(Array) ? spec&.first : spec)
+          return unless (key = update.keys&.first)
           return if key.to_s.start_with?('$')
           raise Error::InvalidUpdateDocument.new(key: key) if Mongo.validate_update_replace
 
@@ -641,8 +641,8 @@ module Mongo
         # @raise [ Error::InvalidUpdateDocument ] if the first key in the
         #   document does not start with a $.
         def validate_replacement_documents!(spec)
-          return unless replace = spec.is_a?(Array) ? spec&.first : spec
-          return unless key = replace.keys&.first
+          return unless (replace = spec.is_a?(Array) ? spec&.first : spec)
+          return unless (key = replace.keys&.first)
           return unless key.to_s.start_with?('$')
           raise Error::InvalidReplacementDocument.new(key: key) if Mongo.validate_update_replace
 

data/lib/mongo/collection/view.rb

@@ -236,6 +236,14 @@ module Mongo
         end
       end
 
+      # Executes the provided block within the context of a session, using
+      # this view's options merged with the given ones.
+      #
+      # @api private
+      def with_session(opts = {}, &block)
+        client.with_session(@options.merge(opts), &block)
+      end
+
       private
 
       def initialize_copy(other)
@@ -252,10 +260,6 @@ module Mongo
       def view
         self
       end
-
-      def with_session(opts = {}, &block)
-        client.with_session(@options.merge(opts), &block)
-      end
     end
   end
 end

data/lib/mongo/collection.rb

@@ -393,7 +393,7 @@ module Mongo
       operation = { create: name }.merge(options)
       operation.delete(:write)
       operation.delete(:write_concern)
-      client.send(:with_session, opts) do |session|
+      client.with_session(opts) do |session|
         write_concern = if opts[:write_concern]
                           WriteConcern.get(opts[:write_concern])
                         else

data/lib/mongo/config.rb

@@ -29,6 +29,11 @@ module Mongo
     # decryption instead of BSON types.
     option :csfle_convert_to_ruby_types, default: false
 
+    # When this flag is set to true, the (host:port) of the server that produced
+    # the error is appended to error messages for OperationFailure and
+    # BulkWriteError. See RUBY-3602.
+    option :include_server_address_in_errors, default: false
+
     # Set the configuration options.
     #
     # @example Set the options.

data/lib/mongo/crypt/auto_encrypter.rb

@@ -287,7 +287,7 @@ module Mongo
         @internal_client ||= client.with(
           auto_encryption_options: nil,
           min_pool_size: 0,
-          monitoring: client.send(:monitoring)
+          monitoring: client.monitoring
         )
       end
     end

data/lib/mongo/crypt/binary.rb

@@ -139,14 +139,17 @@ module Mongo
       # Wraps a String with a mongocrypt_binary_t, yielding an FFI::Pointer
       # to the wrapped struct.
       def self.wrap_string(str)
-        binary_p = Binding.mongocrypt_binary_new_from_data(
-          FFI::MemoryPointer.from_string(str),
-          str.bytesize
-        )
+        # mongocrypt_binary_new_from_data does not copy the buffer, so the
+        # MemoryPointer must outlive the mongocrypt_binary_t. Hold it in a
+        # local so it stays referenced for the whole method frame; otherwise
+        # GC can free the buffer while libmongocrypt still points into it.
+        data_p = FFI::MemoryPointer.from_string(str)
+        binary_p = Binding.mongocrypt_binary_new_from_data(data_p, str.bytesize)
         begin
           yield binary_p
         ensure
           Binding.mongocrypt_binary_destroy(binary_p)
+          data_p # rubocop:disable Lint/Void -- keep the buffer alive past destroy
         end
       end
     end

data/lib/mongo/crypt/binding.rb

@@ -106,7 +106,7 @@ module Mongo
       def self.parse_version(version)
         Gem::Version.new(version)
       rescue ArgumentError
-        match = version.match(/\A(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)?(-[A-Za-z+\d]+)?\z/)
+        match = version.match(/\A(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)?(?:-[A-Za-z+\d]+)?\z/)
         raise ArgumentError.new("Malformed version number string #{version}") if match.nil?
 
         Gem::Version.new(
@@ -823,14 +823,15 @@ module Mongo
 
       # An enum labeling different libmognocrypt state machine states
       enum :mongocrypt_ctx_state, [
-        :error,                 0,
-        :need_mongo_collinfo,   1,
-        :need_mongo_markings,   2,
-        :need_mongo_keys,       3,
-        :need_kms,              4,
-        :ready,                 5,
-        :done,                  6,
-        :need_kms_credentials,  7,
+        :error,                        0,
+        :need_mongo_collinfo,          1,
+        :need_mongo_markings,          2,
+        :need_mongo_keys,              3,
+        :need_kms,                     4,
+        :ready,                        5,
+        :done,                         6,
+        :need_kms_credentials,         7,
+        :need_mongo_collinfo_with_db,  8,
       ]
 
       # @!method self.mongocrypt_ctx_state(ctx)
@@ -1105,10 +1106,20 @@ module Mongo
       def self.check_kms_ctx_status(kms_context)
         return if yield
 
+        kms_ctx_status(kms_context).raise_crypt_error(kms: true)
+      end
+
+      # Read the status information for the given KmsContext into a Status
+      # object without raising.
+      #
+      # @param [ Mongo::Crypt::KmsContext ] kms_context
+      #
+      # @return [ Mongo::Crypt::Status ] The KMS context status.
+      def self.kms_ctx_status(kms_context)
         status = Status.new
 
         mongocrypt_kms_ctx_status(kms_context.kms_ctx_p, status.ref)
-        status.raise_crypt_error(kms: true)
+        status
       end
 
       # @!method self.mongocrypt_kms_ctx_usleep(ctx)
@@ -1646,6 +1657,73 @@ module Mongo
         mongocrypt_setopt_use_need_kms_credentials_state(handle.ref)
       end
 
+      # @!method self.mongocrypt_setopt_enable_multiple_collinfo(crypt)
+      #   @api private
+      #
+      # Enable support for multiple collection schemas. Required to support $lookup.
+      #
+      # @param [ FFI::Pointer ] crypt A pointer to a mongocrypt_t object.
+      # @pre mongocrypt_init has not been called on crypt.
+      # @return [ Boolean ] Whether the option was set successfully.
+      attach_function(
+        :mongocrypt_setopt_enable_multiple_collinfo,
+        [ :pointer ],
+        :bool
+      )
+
+      # Enable support for multiple collection schemas. Required to support $lookup.
+      #
+      # @param [ Mongo::Crypt::Handle ] handle
+      # @return [ Boolean ] Whether the option was set successfully.
+      def self.setopt_enable_multiple_collinfo(handle)
+        check_status(handle) do
+          mongocrypt_setopt_enable_multiple_collinfo(handle.ref)
+        end
+      end
+
+      # @!method self.mongocrypt_setopt_use_need_mongo_collinfo_with_db_state(crypt)
+      #   @api private
+      #
+      # Opt into handling the MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB state.
+      # A context enters this state when processing a bulkWrite command whose
+      # target database differs from the command database ("admin").
+      #
+      # @param [ FFI::Pointer ] crypt A pointer to a mongocrypt_t object.
+      attach_function(
+        :mongocrypt_setopt_use_need_mongo_collinfo_with_db_state,
+        [ :pointer ],
+        :void
+      )
+
+      # Opt into handling the MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB state.
+      #
+      # @param [ Mongo::Crypt::Handle ] handle
+      def self.setopt_use_need_mongo_collinfo_with_db_state(handle)
+        mongocrypt_setopt_use_need_mongo_collinfo_with_db_state(handle.ref)
+      end
+
+      # @!method self.mongocrypt_ctx_mongo_db(ctx)
+      #   @api private
+      #
+      # Get the database name for the current collinfo operation. Used in the
+      # MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB state to determine which
+      # database to run listCollections against.
+      #
+      # @param [ FFI::Pointer ] ctx A pointer to a mongocrypt_ctx_t object.
+      # @return [ String, nil ] The database name, or nil on error.
+      attach_function :mongocrypt_ctx_mongo_db, [ :pointer ], :string
+
+      # Get the database name for the current collinfo operation.
+      #
+      # @param [ Mongo::Crypt::Context ] context
+      # @return [ String ] The database name.
+      def self.ctx_mongo_db(context)
+        db_name = mongocrypt_ctx_mongo_db(context.ctx_p)
+        return db_name if db_name
+
+        check_ctx_status(context)
+      end
+
       # @!method self.mongocrypt_ctx_provide_kms_providers(ctx, kms_providers)
       #   @api private
       #

data/lib/mongo/crypt/context.rb

@@ -76,7 +76,7 @@ module Mongo
       # This method is not currently unit tested. It is integration tested
       # in spec/integration/explicit_encryption_spec.rb
       def run_state_machine(timeout_holder)
-        while true
+        loop do
           timeout_ms = timeout_holder.remaining_timeout_ms!
           case state
           when :error
@@ -90,6 +90,8 @@ module Mongo
             provide_keys(timeout_ms)
           when :need_mongo_collinfo
             provide_collection_info(timeout_ms)
+          when :need_mongo_collinfo_with_db
+            provide_collection_info_with_db(timeout_ms)
           when :need_mongo_markings
             provide_markings(timeout_ms)
           when :need_kms
@@ -122,17 +124,26 @@ module Mongo
       def provide_markings(timeout_ms)
         cmd = Binding.ctx_mongo_op(self)
 
-        result = @encryption_io.mark_command(cmd, timeout_ms: timeout_ms)
+        result = @encryption_io.mark_command(cmd, db_name: @db_name, timeout_ms: timeout_ms)
         mongocrypt_feed(result)
 
         mongocrypt_done
       end
 
       def provide_collection_info(timeout_ms)
+        feed_collection_info(@db_name, timeout_ms)
+      end
+
+      def provide_collection_info_with_db(timeout_ms)
+        feed_collection_info(Binding.ctx_mongo_db(self), timeout_ms)
+      end
+
+      def feed_collection_info(db_name, timeout_ms)
         filter = Binding.ctx_mongo_op(self)
 
-        result = @encryption_io.collection_info(@db_name, filter, timeout_ms: timeout_ms)
-        mongocrypt_feed(result) if result
+        @encryption_io.collection_info(db_name, filter, timeout_ms: timeout_ms).each do |result|
+          mongocrypt_feed(result)
+        end
 
         mongocrypt_done
       end
@@ -159,12 +170,29 @@ module Mongo
             raise unless e.network_error?
             next if Binding.kms_ctx_fail(kms_context)
 
-            raise
+            raise_kms_retry_error(kms_context, e)
           end
         end
         Binding.ctx_kms_done(self)
       end
 
+      # Raise a KmsError that wraps the KMS status message (which describes the
+      # retry exhaustion) with the error from the last attempt.
+      #
+      # @param [ Mongo::Crypt::KmsContext ] kms_context
+      # @param [ Mongo::Error::KmsError ] last_error The error from the last
+      #   KMS request attempt.
+      #
+      # @raise [ Mongo::Error::KmsError ]
+      def raise_kms_retry_error(kms_context, last_error)
+        status = Binding.kms_ctx_status(kms_context)
+        raise Error::KmsError.new(
+          "#{status.message}, last attempt failed with: #{last_error.message}",
+          code: status.code,
+          network_error: true
+        )
+      end
+
       # Indicate that state machine is done feeding I/O responses back to libmongocrypt
       def mongocrypt_done
         Binding.mongocrypt_ctx_mongo_done(ctx_p)
@@ -188,21 +216,24 @@ module Mongo
       #   KMS providers.
       def retrieve_kms_credentials(timeout_holder)
         providers = {}
-        if kms_providers.aws && kms_providers.aws.empty?
-          begin
-            aws_credentials = Mongo::Auth::Aws::CredentialsRetriever.new.credentials(timeout_holder)
-          rescue Auth::Aws::CredentialsNotFound
-            raise Error::CryptError.new(
-              'Could not locate AWS credentials (checked environment variables, ECS and EC2 metadata)'
-            )
+        kms_providers.credentials_map.each do |identifier, creds|
+          next unless creds.empty?
+
+          case KMS.provider_base_type(identifier)
+          when 'aws'
+            begin
+              aws_credentials = Mongo::Auth::Aws::CredentialsRetriever.new.credentials(timeout_holder)
+            rescue Auth::Aws::CredentialsNotFound
+              raise Error::CryptError.new(
+                'Could not locate AWS credentials (checked environment variables, ECS and EC2 metadata)'
+              )
+            end
+            providers[identifier] = aws_credentials.to_h
+          when 'gcp'
+            providers[identifier] = { access_token: gcp_access_token(timeout_holder) }
+          when 'azure'
+            providers[identifier] = { access_token: azure_access_token(timeout_holder) }
           end
-          providers[:aws] = aws_credentials.to_h
-        end
-        if kms_providers.gcp && kms_providers.gcp.empty?
-          providers[:gcp] = { access_token: gcp_access_token(timeout_holder) }
-        end
-        if kms_providers.azure && kms_providers.azure.empty?
-          providers[:azure] = { access_token: azure_access_token(timeout_holder) }
         end
         KMS::Credentials.new(providers)
       end

data/lib/mongo/crypt/encryption_io.rb

@@ -97,7 +97,7 @@ module Mongo
       #    Must be a non-negative integer. An explicit value of 0 means infinite.
       #    The default value is unset which means the feature is not enabled.
       #
-      # @return [ Hash ] The collection information
+      # @return [ Array<BSON::Document> ] The collection information documents
       def collection_info(db_name, filter, timeout_ms: nil)
         unless @metadata_client
           raise ArgumentError,
@@ -108,18 +108,22 @@ module Mongo
           .use(db_name)
           .database
           .list_collections(filter: filter, deserialize_as_bson: true, timeout_ms: timeout_ms)
-          .first
+          .to_a
       end
 
       # Send the command to mongocryptd to be marked with intent-to-encrypt markings
       #
       # @param [ Hash ] cmd
+      # @param [ String | nil ] :db_name The database against which the command
+      #   is being run. When provided, the command is sent to mongocryptd using
+      #   this database so that the namespace in the command matches the namespace
+      #   in encryptionInformation.
       # @param [ Integer ] :timeout_ms The operation timeout in milliseconds.
       #    Must be a non-negative integer. An explicit value of 0 means infinite.
       #    The default value is unset which means the feature is not enabled.
       #
       # @return [ Hash ] The marked command
-      def mark_command(cmd, timeout_ms: nil)
+      def mark_command(cmd, db_name: nil, timeout_ms: nil)
         unless @mongocryptd_client
           raise ArgumentError,
                 'mark_command requires mongocryptd_client to have been passed to the constructor, but it was not'
@@ -132,13 +136,15 @@ module Mongo
           timeout_ms: timeout_ms
         }
 
+        mongocryptd = db_name ? @mongocryptd_client.use(db_name) : @mongocryptd_client
+
         begin
-          response = @mongocryptd_client.database.command(cmd, options)
+          response = mongocryptd.database.command(cmd, options)
         rescue Error::NoServerAvailable => e
           raise e if @options[:mongocryptd_bypass_spawn]
 
           spawn_mongocryptd
-          response = @mongocryptd_client.database.command(cmd, options)
+          response = mongocryptd.database.command(cmd, options)
         end
 
         response.first

data/lib/mongo/crypt/handle.rb

@@ -74,6 +74,8 @@ module Mongo
           Binding.method(:mongocrypt_destroy)
         )
         Binding.kms_ctx_setopt_retry_kms(self, true)
+        Binding.setopt_enable_multiple_collinfo(self)
+        Binding.setopt_use_need_mongo_collinfo_with_db_state(self)
         @kms_providers = kms_providers
         @kms_tls_options = kms_tls_options
 
@@ -101,7 +103,7 @@ module Mongo
 
         Binding.setopt_kms_providers(self, @kms_providers.to_document)
 
-        if @kms_providers.aws&.empty? || @kms_providers.gcp&.empty? || @kms_providers.azure&.empty?
+        if @kms_providers.any_on_demand?
           Binding.setopt_use_need_kms_credentials_state(self)
         end
 
@@ -123,13 +125,23 @@ module Mongo
       end
 
       # Return TLS options for KMS provider. If there are no TLS options set,
-      # empty hash is returned.
+      # empty hash is returned. Named providers (e.g. "kmip:name1") fall back
+      # to the base-type options (e.g. :kmip) when no exact match is found.
       #
-      # @param [ String ] provider KSM provider name.
+      # @param [ String ] provider KMS provider name or named identifier.
       #
       # @return [ Hash ] TLS options to connect to KMS provider.
       def kms_tls_options(provider)
-        @kms_tls_options.fetch(provider, {})
+        provider_str = provider.to_s
+        base_type = KMS.provider_base_type(provider_str)
+
+        @kms_tls_options.fetch(provider_str) do
+          @kms_tls_options.fetch(provider_str.to_sym) do
+            @kms_tls_options.fetch(base_type) do
+              @kms_tls_options.fetch(base_type.to_sym, {})
+            end
+          end
+        end
       end
 
       def crypt_shared_lib_version
@@ -191,7 +203,7 @@ module Mongo
       # Send the logs from libmongocrypt to the Mongo::Logger
       def set_logger_callback
         @log_callback = proc do |level, msg|
-          @logger.send(level, msg)
+          @logger.public_send(level, msg)
         end
 
         Binding.setopt_log_handler(@mongocrypt, @log_callback)