mongo 2.17.4 → 2.18.0.beta1

data/lib/mongo/client_encryption.rb

@@ -31,13 +31,24 @@ module Mongo
     # @option options [ String ] :key_vault_namespace The name of the
     #   key vault collection in the format "database.collection".
     # @option options [ Hash ] :kms_providers A hash of key management service
-    #   configuration information. Valid hash keys are :local or :aws. There
-    #   may be more than one KMS provider specified.
+    #   configuration information.
+    #   @see Mongo::Crypt::KMS::Credentials for list of options for every
+    #   supported provider.
+    #   @note There may be more than one KMS provider specified.
+    # @option options [ Hash ] :kms_tls_options TLS options to connect to KMS
+    #   providers. Keys of the hash should be KSM provider names; values
+    #   should be hashes of TLS connection options. The options are equivalent
+    #   to TLS connection options of Mongo::Client.
+    #   @see Mongo::Client#initialize for list of TLS options.
+    #
+    # @raise [ ArgumentError ] If required options are missing or incorrectly
+    #   formatted.
     def initialize(key_vault_client, options={})
       @encrypter = Crypt::ExplicitEncrypter.new(
         key_vault_client,
         options[:key_vault_namespace],
-        options[:kms_providers]
+        Crypt::KMS::Credentials.new(options[:kms_providers]),
+        Crypt::KMS::Validations.validate_tls_options(options[:kms_tls_options])
       )
     end
 
@@ -64,10 +75,9 @@ module Mongo
     # @return [ BSON::Binary ] The 16-byte UUID of the new data key as a
     #   BSON::Binary object with type :uuid.
     def create_data_key(kms_provider, options={})
-      @encrypter.create_and_insert_data_key(
-        kms_provider,
-        options
-      )
+      key_document = Crypt::KMS::MasterKeyDocument.new(kms_provider, options)
+      key_alt_names = options[:key_alt_names]
+      @encrypter.create_and_insert_data_key(key_document, key_alt_names)
     end
 
     # Encrypts a value using the specified encryption key and algorithm.
@@ -81,14 +91,25 @@ module Mongo
     # @option options [ String ] :key_alt_name The alternate name for the
     #   encryption key.
     # @option options [ String ] :algorithm The algorithm used to encrypt the value.
-    #   Valid algorithms are "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
-    #   or "AEAD_AES_256_CBC_HMAC_SHA_512-Random".
+    #   Valid algorithms are "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic",
+    #   "AEAD_AES_256_CBC_HMAC_SHA_512-Random", "Indexed", "Unindexed".
+    # @option options [ Integer | nil ] :contention_factor Contention factor
+    #   to be applied if encryption algorithm is set to "Indexed". If not
+    #   provided, it defaults to a value of 0. Contention factor should be set
+    #   only if encryption algorithm is set to "Indexed".
+    # @option options [ Symbol ] query_type Query type to be applied
+    # if encryption algorithm is set to "Indexed". Query type should be set
+    #   only if encryption algorithm is set to "Indexed". The only allowed
+    #   value is :equality.
     #
     # @note The :key_id and :key_alt_name options are mutually exclusive. Only
     #   one is required to perform explicit encryption.
     #
     # @return [ BSON::Binary ] A BSON Binary object of subtype 6 (ciphertext)
     #   representing the encrypted value.
+    #
+    # @raise [ ArgumentError ] if either contention_factor or query_type
+    #   is set, and algorithm is not "Indexed".
     def encrypt(value, options={})
       @encrypter.encrypt(value, options)
     end

data/lib/mongo/cluster/reapers/cursor_reaper.rb

@@ -192,9 +192,19 @@ module Mongo
 
           options = {
             server_api: server.options[:server_api],
-            service_id: kill_spec.service_id,
+            connection_global_id: kill_spec.connection_global_id,
           }
           op.execute(server, context: Operation::Context.new(options: options))
+
+          if session = kill_spec.session
+            if session.ended?
+              if Lint.enabled?
+                raise Error::LintError, "Ended session for a garbage collected cursor"
+              end
+            elsif session.implicit?
+              session.end_session
+            end
+          end
         end
       end
       alias :execute :kill_cursors

data/lib/mongo/cluster.rb

@@ -937,11 +937,21 @@ module Mongo
     # is connected to does not support sessions.
     #
     # Session support may change over time, for example due to servers in the
-    # deployment being upgraded or downgraded. If the client is currently not
-    # connected to any data bearing servers, this method considers the state
-    # of session support as of when the client was last connected to at
-    # least one server. If the client has never connected to any servers,
-    # the deployment is considered to not support sessions.
+    # deployment being upgraded or downgraded.  If the client isn't connected to
+    # any servers and doesn't find any servers
+    # for the duration of server selection timeout, this method will raise
+    # NoServerAvailable. This method is called from the operation execution flow,
+    # and if it raises NoServerAvailable the entire operation will fail
+    # with that exception, since the operation execution has waited for
+    # the server selection timeout for any server to become available
+    # (which would be a superset of the servers suitable for the operation being
+    # attempted) and none materialized.
+    #
+    # @raise [ Error::SessionsNotAvailable ] If the deployment that the driver
+    #   is connected to does not support sessions.
+    # @raise [ Error::NoServerAvailable ] If the client isn't connected to
+    #   any servers and doesn't find any servers for the duration of
+    #   server selection timeout.
     #
     # @api private
     def validate_session_support!
@@ -962,25 +972,11 @@ module Mongo
       # No data bearing servers known - perform server selection to try to
       # get a response from at least one of them, to return an accurate
       # assessment of whether sessions are currently supported.
-      begin
-        ServerSelector.get(mode: :primary_preferred).select_server(self)
-        @state_change_lock.synchronize do
-          @sdam_flow_lock.synchronize do
-            unless topology.logical_session_timeout
-              raise_sessions_not_supported
-            end
-          end
-        end
-      rescue Error::NoServerAvailable
-        # We haven't been able to contact any servers - use last known
-        # value for session support.
-        @state_change_lock.synchronize do
-          @sdam_flow_lock.synchronize do
-            @update_lock.synchronize do
-              unless @sessions_supported
-                raise_sessions_not_supported
-              end
-            end
+      ServerSelector.get(mode: :primary_preferred).select_server(self)
+      @state_change_lock.synchronize do
+        @sdam_flow_lock.synchronize do
+          unless topology.logical_session_timeout
+            raise_sessions_not_supported
           end
         end
       end

data/lib/mongo/collection/helpers.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2022 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Collection
+    # This module contains helper methods collection class.
+    #
+    # @api private
+    module Helpers
+      # Executes drop operation and and ignores NamespaceNotFound error.
+      #
+      # @param [ Operation::Drop ] operation Drop operation to be executed.
+      # @param [ Session ] session Session to be use for execution.
+      # @param [ Operation::Context ] context Context to use for execution.
+      #
+      # @return [ Result ] The result of the execution.
+      def do_drop(operation, session, context)
+        operation.execute(next_primary(nil, session), context: context)
+      rescue Error::OperationFailure => ex
+        # NamespaceNotFound
+        if ex.code == 26 || ex.code.nil? && ex.message =~ /ns not found/
+          false
+        else
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/mongo/collection/queryable_encryption.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2022 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Collection
+    # This module contains methods for creating and dropping auxiliary collections
+    # for queryable encryption.
+    #
+    # @api private
+    module QueryableEncryption
+
+      # Creates auxiliary collections and indices for queryable encryption if necessary.
+      #
+      # @param [ Hash | nil ] encrypted_fields Encrypted fields hash that was
+      #   provided to `create` collection helper.
+      # @param [ Client ] client Mongo client to be used to create auxiliary collections.
+      # @param [ Session ] session Session to be used to create auxiliary collections.
+      #
+      # @return [ Result ] The result of provided block.
+      def maybe_create_qe_collections(encrypted_fields, client, session)
+        encrypted_fields = if encrypted_fields
+          encrypted_fields
+        elsif encrypted_fields_map
+          encrypted_fields_map[namespace] || {}
+        else
+          {}
+        end
+        if encrypted_fields.empty?
+          return yield
+        end
+
+        emm_collections(encrypted_fields).each do |coll|
+          context = Operation::Context.new(client: client, session: session)
+          Operation::Create.new(
+            selector: {
+              create: coll,
+              clusteredIndex: {
+                key: {
+                  _id: 1
+                },
+                unique: true
+              }
+            },
+            db_name: database.name,
+          ).execute(next_primary(nil, session), context: context)
+        end
+        yield(encrypted_fields).tap do |result|
+          indexes.create_one(__safeContent__: 1) if result
+        end
+      end
+
+      # Drops auxiliary collections and indices for queryable encryption if necessary.
+      #
+      # @param [ Hash | nil ] encrypted_fields Encrypted fields hash that was
+      #   provided to `create` collection helper.
+      # @param [ Client ] client Mongo client to be used to drop auxiliary collections.
+      # @param [ Session ] session Session to be used to drop auxiliary collections.
+      #
+      # @return [ Result ] The result of provided block.
+      def maybe_drop_emm_collections(encrypted_fields, client, session)
+        encrypted_fields = if encrypted_fields
+          encrypted_fields
+        elsif encrypted_fields_map
+          if encrypted_fields_map[namespace]
+            encrypted_fields_map[namespace]
+          else
+            database.list_collections(filter: { name: name })
+              .first
+              &.fetch(:options, {})
+              &.fetch(:encryptedFields, {}) || {}
+          end
+        else
+          {}
+        end
+        if encrypted_fields.empty?
+          return yield
+        end
+        emm_collections(encrypted_fields).each do |coll|
+          context = Operation::Context.new(client: client, session: session)
+          operation = Operation::Drop.new(
+            selector: { drop: coll },
+            db_name: database.name,
+            session: session,
+          )
+          do_drop(operation, session, context)
+        end
+        yield
+      end
+
+      private
+
+      # Checks if names for auxiliary collections are set and returns them,
+      # otherwise returns default names.
+      #
+      # @param [ Hash ] encrypted_fields Encrypted fields hash.
+      #
+      # @return [ Array <String> ] Array of auxiliary collections names.
+      def emm_collections(encrypted_fields)
+        [
+          encrypted_fields['escCollection'] || "enxcol_.#{name}.esc",
+          encrypted_fields['eccCollection'] || "enxcol_.#{name}.ecc",
+          encrypted_fields['ecocCollection'] || "enxcol_.#{name}.ecoc",
+        ]
+      end
+
+    end
+  end
+end

data/lib/mongo/collection/view/aggregation.rb

@@ -80,7 +80,8 @@ module Mongo
         # @option options [ true, false ] :bypass_document_validation Whether or
         #   not to skip document level validation.
         # @option options [ Hash ] :collation The collation to use.
-        # @option options [ String ] :comment Associate a comment with the aggregation.
+        # @option options [ Object ] :comment A user-provided
+        #   comment to attach to this command.
         # @option options [ String ] :hint The index to use for the aggregation.
         # @option options [ Hash ] :let Mapping of variables to use in the pipeline.
         #   See the server documentation for details.
@@ -127,7 +128,7 @@ module Mongo
           @view.send(:server_selector)
         end
 
-        def aggregate_spec(server, session, read_preference)
+        def aggregate_spec(session, read_preference)
           Builder::Aggregation.new(
             pipeline,
             view,
@@ -139,34 +140,35 @@ module Mongo
           Aggregation.new(view, pipeline, options)
         end
 
-        def initial_query_op(server, session, read_preference)
-          Operation::Aggregate.new(aggregate_spec(server, session, read_preference))
+        def initial_query_op(session, read_preference)
+          Operation::Aggregate.new(aggregate_spec(session, read_preference))
         end
 
         # Return effective read preference for the operation.
         #
         # If the pipeline contains $merge or $out, and read preference specified
-        # by user is secondary or secondary_preferred, and selected server is below
+        # by user is secondary or secondary_preferred, and target server is below
         # 5.0, than this method returns primary read preference, because the
         # aggregation will be routed to primary. Otherwise return the original
         # read preference.
         #
         # See https://github.com/mongodb/specifications/blob/master/source/crud/crud.rst#read-preferences-and-server-selection
         #
-        # @param [ Server ] server The server on which the operation
-        #   should be executed.
+        # @param [ Server::Connection ] connection The connection which
+        #   will be used for the operation.
         # @return [ Hash | nil ] read preference hash that should be sent with
         #   this command.
-        def effective_read_preference(server)
+        def effective_read_preference(connection)
           return unless view.read_preference
           return view.read_preference unless write?
           return view.read_preference unless [:secondary, :secondary_preferred].include?(view.read_preference[:mode])
 
           primary_read_preference = {mode: :primary}
-          if server.primary?
+          description = connection.description
+          if description.primary?
             log_warn("Routing the Aggregation operation to the primary server")
             primary_read_preference
-          elsif server.mongos? && !server.features.merge_out_on_secondary_enabled?
+          elsif description.mongos? && !description.features.merge_out_on_secondary_enabled?
             log_warn("Routing the Aggregation operation to the primary server")
             primary_read_preference
           else
@@ -176,14 +178,15 @@ module Mongo
         end
 
         def send_initial_query(server, session)
-          initial_query_op(
-            server,
-            session,
-            effective_read_preference(server)
-          ).execute(
-              server,
+          server.with_connection do |connection|
+            initial_query_op(
+              session,
+              effective_read_preference(connection)
+            ).execute_with_connection(
+              connection,
               context: Operation::Context.new(client: client, session: session)
             )
+          end
         end
 
         # Skip, sort, limit, projection are specified as pipeline stages

data/lib/mongo/collection/view/change_stream.rb

@@ -70,10 +70,35 @@ module Mongo
         # @param [ Array<Hash> ] pipeline The pipeline of operators to filter the change notifications.
         # @param [ Hash ] options The change stream options.
         #
-        # @option options [ String ] :full_document Allowed values: 'default', 'updateLookup'. Defaults to 'default'.
-        #   When set to 'updateLookup', the change notification for partial updates will include both a delta
-        #   describing the changes to the document, as well as a copy of the entire document that was changed
-        #   from some time after the change occurred.
+        # @option options [ String ] :full_document Allowed values: nil, 'default',
+        #   'updateLookup', 'whenAvailable', 'required'.
+        #
+        #   The default is to not send a value (i.e. nil), which is equivalent to
+        #   'default'. By default, the change notification for partial updates will
+        #   include a delta describing the changes to the document.
+        #
+        #   When set to 'updateLookup', the change notification for partial updates
+        #   will include both a delta describing the changes to the document as well
+        #   as a copy of the entire document that was changed from some time after
+        #   the change occurred.
+        #
+        #   When set to 'whenAvailable', configures the change stream to return the
+        #   post-image of the modified document for replace and update change events
+        #   if the post-image for this event is available.
+        #
+        #   When set to 'required', the same behavior as 'whenAvailable' except that
+        #   an error is raised if the post-image is not available.
+        # @option options [ String ] :full_document_before_change Allowed values: nil,
+        #   'whenAvailable', 'required', 'off'.
+        #
+        #   The default is to not send a value (i.e. nil), which is equivalent to 'off'.
+        #
+        #   When set to 'whenAvailable', configures the change stream to return the
+        #   pre-image of the modified document for replace, update, and delete change
+        #   events if it is available.
+        #
+        #   When set to 'required', the same behavior as 'whenAvailable' except that
+        #   an error is raised if the pre-image is not available.
         # @option options [ BSON::Document, Hash ] :resume_after Specifies the logical starting point for the
         #   new change stream.
         # @option options [ Integer ] :max_await_time_ms The maximum amount of time for the server to wait
@@ -88,6 +113,8 @@ module Mongo
         #   option takes a resume token and starts a new change stream returning the first
         #   notification after the token. This will allow users to watch collections that have been
         #   dropped and recreated or newly renamed collections without missing any notifications.
+        # @option options [ Object ] :comment A user-provided
+        #   comment to attach to this command.
         #
         #   The server will report an error if `startAfter` and `resumeAfter` are both specified.
         #
@@ -275,20 +302,22 @@ module Mongo
           start_at_operation_time = nil
           start_at_operation_time_supported = nil
           @cursor = read_with_retry_cursor(session, server_selector, view) do |server|
-            start_at_operation_time_supported = server.description.server_version_gte?('4.0')
+            server.with_connection do |connection|
+              start_at_operation_time_supported = connection.description.server_version_gte?('4.0')
 
-            result = send_initial_query(server, session)
-            if doc = result.replies.first && result.replies.first.documents.first
-              start_at_operation_time = doc['operationTime']
-            else
-              # The above may set @start_at_operation_time to nil
-              # if it was not in the document for some reason,
-              # for consistency set it to nil here as well.
-              # NB: since this block may be executed more than once, each
-              # execution must write to start_at_operation_time either way.
-              start_at_operation_time = nil
+              result = send_initial_query(connection, session)
+              if doc = result.replies.first && result.replies.first.documents.first
+                start_at_operation_time = doc['operationTime']
+              else
+                # The above may set @start_at_operation_time to nil
+                # if it was not in the document for some reason,
+                # for consistency set it to nil here as well.
+                # NB: since this block may be executed more than once, each
+                # execution must write to start_at_operation_time either way.
+                start_at_operation_time = nil
+              end
+              result
             end
-            result
           end
           @start_at_operation_time = start_at_operation_time
           @start_at_operation_time_supported = start_at_operation_time_supported
@@ -298,8 +327,8 @@ module Mongo
           [{ '$changeStream' => change_doc }] + @change_stream_filters
         end
 
-        def aggregate_spec(server, session, read_preference)
-          super(server, session, read_preference).tap do |spec|
+        def aggregate_spec(session, read_preference)
+          super(session, read_preference).tap do |spec|
             spec[:selector][:aggregate] = 1 unless for_collection?
           end
         end
@@ -310,6 +339,10 @@ module Mongo
               doc[:fullDocument] = @options[:full_document]
             end
 
+            if @options[:full_document_before_change]
+              doc[:fullDocumentBeforeChange] = @options[:full_document_before_change]
+            end
+
             if resuming?
               # We have a resume token once we retrieved any documents.
               # However, if the first getMore fails and the user didn't pass
@@ -348,11 +381,11 @@ module Mongo
           end
         end
 
-        def send_initial_query(server, session)
-          initial_query_op(server, session, view.read_preference)
-            .execute(
-              server,
-              context: Operation::Context.new(client: client, session: session)
+        def send_initial_query(connection, session)
+          initial_query_op(session, view.read_preference)
+            .execute_with_connection(
+              connection,
+              context: Operation::Context.new(client: client, session: session),
             )
         end
 

data/lib/mongo/collection/view/explainable.rb

@@ -53,7 +53,7 @@ module Mongo
         #
         # @return [ Hash ] A single document with the query plan.
         #
-        # @see https://docs.mongodb.com/manual/reference/method/db.collection.explain/#db.collection.explain
+        # @see https://mongodb.com/docs/manual/reference/method/db.collection.explain/#db.collection.explain
         #
         # @since 2.0.0
         def explain(**opts)

data/lib/mongo/collection/view/iterable.rb

@@ -149,7 +149,7 @@ module Mongo
           }
         end
 
-        def initial_query_op(server, session)
+        def initial_query_op(session)
           spec = {
             coll_name: collection.name,
             filter: filter,
@@ -170,7 +170,6 @@ module Mongo
             max_time_ms: options[:max_time_ms],
             max_value: options[:max_value],
             min_value: options[:min_value],
-            no_cursor_timeout: options[:no_cursor_timeout],
             return_key: options[:return_key],
             show_disk_loc: options[:show_disk_loc],
             comment: options[:comment],
@@ -185,8 +184,6 @@ module Mongo
             collection.client.log_warn("The :oplog_replay option is deprecated and ignored by MongoDB 4.4 and later")
           end
 
-          maybe_set_tailable_options(spec)
-
           if explained?
             spec[:explain] = options[:explain]
             Operation::Explain.new(spec)
@@ -196,25 +193,12 @@ module Mongo
         end
 
         def send_initial_query(server, session = nil)
-          initial_query_op(server, session).execute(server, context: Operation::Context.new(client: client, session: session))
+          initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
 
         def use_query_cache?
           QueryCache.enabled? && !collection.system_collection?
         end
-
-        # Add tailable cusror options to the command specifiction if needed.
-        #
-        # @param [ Hash ] spec The command specification.
-        def maybe_set_tailable_options(spec)
-          case cursor_type
-          when :tailable
-            spec[:tailable] = true
-          when :tailable_await
-            spec[:tailable] = true
-            spec[:await_data] = true
-          end
-        end
       end
     end
   end

data/lib/mongo/collection/view/map_reduce.rb

@@ -72,7 +72,7 @@ module Mongo
           @cursor = nil
           session = client.send(:get_session, @options)
           server = cluster.next_primary(nil, session)
-          result = send_initial_query(server, session)
+          result = send_initial_query(server, session, context: Operation::Context.new(client: client, session: session))
           result = send_fetch_query(server, session) unless inline?
           @cursor = Cursor.new(view, result, server, session: session)
           if block_given?
@@ -223,8 +223,9 @@ module Mongo
         def execute
           view.send(:with_session, @options) do |session|
             write_concern = view.write_concern_with_session(session)
-            nro_write_with_retry(session, write_concern) do |server|
-              send_initial_query(server, session)
+            context = Operation::Context.new(client: client, session: session)
+            nro_write_with_retry(write_concern, context: context) do |connection, txn_num, context|
+              send_initial_query_with_connection(connection, session, context: context)
             end
           end
         end
@@ -266,11 +267,10 @@ module Mongo
           Operation::MapReduce.new(spec)
         end
 
-        def valid_server?(server)
+        def valid_server?(description)
           if secondary_ok?
             true
           else
-            description = server.description
             description.standalone? || description.mongos? || description.primary? || description.load_balancer?
           end
         end
@@ -279,13 +279,22 @@ module Mongo
           out.respond_to?(:keys) && out.keys.first.to_s.downcase == INLINE
         end
 
-        def send_initial_query(server, session)
-          unless valid_server?(server)
-            msg = "Rerouting the MapReduce operation to the primary server - #{server.summary} is not suitable"
+        def send_initial_query(server, session, context:)
+          server.with_connection do |connection|
+            send_initial_query_with_connection(connection, session, context: context)
+          end
+        end
+
+        def send_initial_query_with_connection(connection, session, context:)
+          op = initial_query_op(session)
+          if valid_server?(connection.description)
+            op.execute_with_connection(connection, context: context)
+          else
+            msg = "Rerouting the MapReduce operation to the primary server - #{connection.address} is not suitable because it is not currently the primray"
             log_warn(msg)
             server = cluster.next_primary(nil, session)
+            op.execute(server, context: context)
           end
-          initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
 
         def fetch_query_spec