mongo 2.17.2 → 2.18.1

data/lib/mongo/collection/queryable_encryption.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2022 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Collection
+    # This module contains methods for creating and dropping auxiliary collections
+    # for queryable encryption.
+    #
+    # @api private
+    module QueryableEncryption
+
+      # Creates auxiliary collections and indices for queryable encryption if necessary.
+      #
+      # @param [ Hash | nil ] encrypted_fields Encrypted fields hash that was
+      #   provided to `create` collection helper.
+      # @param [ Client ] client Mongo client to be used to create auxiliary collections.
+      # @param [ Session ] session Session to be used to create auxiliary collections.
+      #
+      # @return [ Result ] The result of provided block.
+      def maybe_create_qe_collections(encrypted_fields, client, session)
+        encrypted_fields = if encrypted_fields
+          encrypted_fields
+        elsif encrypted_fields_map
+          encrypted_fields_map[namespace] || {}
+        else
+          {}
+        end
+        if encrypted_fields.empty?
+          return yield
+        end
+
+        emm_collections(encrypted_fields).each do |coll|
+          context = Operation::Context.new(client: client, session: session)
+          Operation::Create.new(
+            selector: {
+              create: coll,
+              clusteredIndex: {
+                key: {
+                  _id: 1
+                },
+                unique: true
+              }
+            },
+            db_name: database.name,
+          ).execute(next_primary(nil, session), context: context)
+        end
+        yield(encrypted_fields).tap do |result|
+          indexes.create_one(__safeContent__: 1) if result
+        end
+      end
+
+      # Drops auxiliary collections and indices for queryable encryption if necessary.
+      #
+      # @param [ Hash | nil ] encrypted_fields Encrypted fields hash that was
+      #   provided to `create` collection helper.
+      # @param [ Client ] client Mongo client to be used to drop auxiliary collections.
+      # @param [ Session ] session Session to be used to drop auxiliary collections.
+      #
+      # @return [ Result ] The result of provided block.
+      def maybe_drop_emm_collections(encrypted_fields, client, session)
+        encrypted_fields = if encrypted_fields
+          encrypted_fields
+        elsif encrypted_fields_map
+          if encrypted_fields_map[namespace]
+            encrypted_fields_map[namespace]
+          else
+            database.list_collections(filter: { name: name })
+              .first
+              &.fetch(:options, {})
+              &.fetch(:encryptedFields, {}) || {}
+          end
+        else
+          {}
+        end
+        if encrypted_fields.empty?
+          return yield
+        end
+        emm_collections(encrypted_fields).each do |coll|
+          context = Operation::Context.new(client: client, session: session)
+          operation = Operation::Drop.new(
+            selector: { drop: coll },
+            db_name: database.name,
+            session: session,
+          )
+          do_drop(operation, session, context)
+        end
+        yield
+      end
+
+      private
+
+      # Checks if names for auxiliary collections are set and returns them,
+      # otherwise returns default names.
+      #
+      # @param [ Hash ] encrypted_fields Encrypted fields hash.
+      #
+      # @return [ Array <String> ] Array of auxiliary collections names.
+      def emm_collections(encrypted_fields)
+        [
+          encrypted_fields['escCollection'] || "enxcol_.#{name}.esc",
+          encrypted_fields['eccCollection'] || "enxcol_.#{name}.ecc",
+          encrypted_fields['ecocCollection'] || "enxcol_.#{name}.ecoc",
+        ]
+      end
+
+    end
+  end
+end

data/lib/mongo/collection/view/aggregation.rb

@@ -80,7 +80,8 @@ module Mongo
         # @option options [ true, false ] :bypass_document_validation Whether or
         #   not to skip document level validation.
         # @option options [ Hash ] :collation The collation to use.
-        # @option options [ String ] :comment Associate a comment with the aggregation.
+        # @option options [ Object ] :comment A user-provided
+        #   comment to attach to this command.
         # @option options [ String ] :hint The index to use for the aggregation.
         # @option options [ Hash ] :let Mapping of variables to use in the pipeline.
         #   See the server documentation for details.
@@ -96,6 +97,9 @@ module Mongo
         def initialize(view, pipeline, options = {})
           @view = view
           @pipeline = pipeline.dup
+          unless Mongo.broken_view_aggregate || view.filter.empty?
+            @pipeline.unshift(:$match => view.filter)
+          end
           @options = BSON::Document.new(options).freeze
         end
 
@@ -127,7 +131,7 @@ module Mongo
           @view.send(:server_selector)
         end
 
-        def aggregate_spec(server, session, read_preference)
+        def aggregate_spec(session, read_preference)
           Builder::Aggregation.new(
             pipeline,
             view,
@@ -139,34 +143,35 @@ module Mongo
           Aggregation.new(view, pipeline, options)
         end
 
-        def initial_query_op(server, session, read_preference)
-          Operation::Aggregate.new(aggregate_spec(server, session, read_preference))
+        def initial_query_op(session, read_preference)
+          Operation::Aggregate.new(aggregate_spec(session, read_preference))
         end
 
         # Return effective read preference for the operation.
         #
         # If the pipeline contains $merge or $out, and read preference specified
-        # by user is secondary or secondary_preferred, and selected server is below
+        # by user is secondary or secondary_preferred, and target server is below
         # 5.0, than this method returns primary read preference, because the
         # aggregation will be routed to primary. Otherwise return the original
         # read preference.
         #
         # See https://github.com/mongodb/specifications/blob/master/source/crud/crud.rst#read-preferences-and-server-selection
         #
-        # @param [ Server ] server The server on which the operation
-        #   should be executed.
+        # @param [ Server::Connection ] connection The connection which
+        #   will be used for the operation.
         # @return [ Hash | nil ] read preference hash that should be sent with
         #   this command.
-        def effective_read_preference(server)
+        def effective_read_preference(connection)
           return unless view.read_preference
           return view.read_preference unless write?
           return view.read_preference unless [:secondary, :secondary_preferred].include?(view.read_preference[:mode])
 
           primary_read_preference = {mode: :primary}
-          if server.primary?
+          description = connection.description
+          if description.primary?
             log_warn("Routing the Aggregation operation to the primary server")
             primary_read_preference
-          elsif server.mongos? && !server.features.merge_out_on_secondary_enabled?
+          elsif description.mongos? && !description.features.merge_out_on_secondary_enabled?
             log_warn("Routing the Aggregation operation to the primary server")
             primary_read_preference
           else
@@ -176,14 +181,15 @@ module Mongo
         end
 
         def send_initial_query(server, session)
-          initial_query_op(
-            server,
-            session,
-            effective_read_preference(server)
-          ).execute(
-              server,
+          server.with_connection do |connection|
+            initial_query_op(
+              session,
+              effective_read_preference(connection)
+            ).execute_with_connection(
+              connection,
               context: Operation::Context.new(client: client, session: session)
             )
+          end
         end
 
         # Skip, sort, limit, projection are specified as pipeline stages

data/lib/mongo/collection/view/change_stream.rb

@@ -70,10 +70,35 @@ module Mongo
         # @param [ Array<Hash> ] pipeline The pipeline of operators to filter the change notifications.
         # @param [ Hash ] options The change stream options.
         #
-        # @option options [ String ] :full_document Allowed values: 'default', 'updateLookup'. Defaults to 'default'.
-        #   When set to 'updateLookup', the change notification for partial updates will include both a delta
-        #   describing the changes to the document, as well as a copy of the entire document that was changed
-        #   from some time after the change occurred.
+        # @option options [ String ] :full_document Allowed values: nil, 'default',
+        #   'updateLookup', 'whenAvailable', 'required'.
+        #
+        #   The default is to not send a value (i.e. nil), which is equivalent to
+        #   'default'. By default, the change notification for partial updates will
+        #   include a delta describing the changes to the document.
+        #
+        #   When set to 'updateLookup', the change notification for partial updates
+        #   will include both a delta describing the changes to the document as well
+        #   as a copy of the entire document that was changed from some time after
+        #   the change occurred.
+        #
+        #   When set to 'whenAvailable', configures the change stream to return the
+        #   post-image of the modified document for replace and update change events
+        #   if the post-image for this event is available.
+        #
+        #   When set to 'required', the same behavior as 'whenAvailable' except that
+        #   an error is raised if the post-image is not available.
+        # @option options [ String ] :full_document_before_change Allowed values: nil,
+        #   'whenAvailable', 'required', 'off'.
+        #
+        #   The default is to not send a value (i.e. nil), which is equivalent to 'off'.
+        #
+        #   When set to 'whenAvailable', configures the change stream to return the
+        #   pre-image of the modified document for replace, update, and delete change
+        #   events if it is available.
+        #
+        #   When set to 'required', the same behavior as 'whenAvailable' except that
+        #   an error is raised if the pre-image is not available.
         # @option options [ BSON::Document, Hash ] :resume_after Specifies the logical starting point for the
         #   new change stream.
         # @option options [ Integer ] :max_await_time_ms The maximum amount of time for the server to wait
@@ -88,6 +113,13 @@ module Mongo
         #   option takes a resume token and starts a new change stream returning the first
         #   notification after the token. This will allow users to watch collections that have been
         #   dropped and recreated or newly renamed collections without missing any notifications.
+        # @option options [ Object ] :comment A user-provided
+        #   comment to attach to this command.
+        # @option options [ Boolean ] :show_expanded_events Enables the server to
+        #   send the 'expanded' list of change stream events. The list of additional
+        #   events included with this flag set are: createIndexes, dropIndexes,
+        #   modify, create, shardCollection, reshardCollection,
+        #   refineCollectionShardKey.
         #
         #   The server will report an error if `startAfter` and `resumeAfter` are both specified.
         #
@@ -275,20 +307,22 @@ module Mongo
           start_at_operation_time = nil
           start_at_operation_time_supported = nil
           @cursor = read_with_retry_cursor(session, server_selector, view) do |server|
-            start_at_operation_time_supported = server.description.server_version_gte?('4.0')
+            server.with_connection do |connection|
+              start_at_operation_time_supported = connection.description.server_version_gte?('4.0')
 
-            result = send_initial_query(server, session)
-            if doc = result.replies.first && result.replies.first.documents.first
-              start_at_operation_time = doc['operationTime']
-            else
-              # The above may set @start_at_operation_time to nil
-              # if it was not in the document for some reason,
-              # for consistency set it to nil here as well.
-              # NB: since this block may be executed more than once, each
-              # execution must write to start_at_operation_time either way.
-              start_at_operation_time = nil
+              result = send_initial_query(connection, session)
+              if doc = result.replies.first && result.replies.first.documents.first
+                start_at_operation_time = doc['operationTime']
+              else
+                # The above may set @start_at_operation_time to nil
+                # if it was not in the document for some reason,
+                # for consistency set it to nil here as well.
+                # NB: since this block may be executed more than once, each
+                # execution must write to start_at_operation_time either way.
+                start_at_operation_time = nil
+              end
+              result
             end
-            result
           end
           @start_at_operation_time = start_at_operation_time
           @start_at_operation_time_supported = start_at_operation_time_supported
@@ -298,8 +332,8 @@ module Mongo
           [{ '$changeStream' => change_doc }] + @change_stream_filters
         end
 
-        def aggregate_spec(server, session, read_preference)
-          super(server, session, read_preference).tap do |spec|
+        def aggregate_spec(session, read_preference)
+          super(session, read_preference).tap do |spec|
             spec[:selector][:aggregate] = 1 unless for_collection?
           end
         end
@@ -310,6 +344,14 @@ module Mongo
               doc[:fullDocument] = @options[:full_document]
             end
 
+            if @options[:full_document_before_change]
+              doc[:fullDocumentBeforeChange] = @options[:full_document_before_change]
+            end
+
+            if @options.key?(:show_expanded_events)
+              doc[:showExpandedEvents] = @options[:show_expanded_events]
+            end
+
             if resuming?
               # We have a resume token once we retrieved any documents.
               # However, if the first getMore fails and the user didn't pass
@@ -348,11 +390,11 @@ module Mongo
           end
         end
 
-        def send_initial_query(server, session)
-          initial_query_op(server, session, view.read_preference)
-            .execute(
-              server,
-              context: Operation::Context.new(client: client, session: session)
+        def send_initial_query(connection, session)
+          initial_query_op(session, view.read_preference)
+            .execute_with_connection(
+              connection,
+              context: Operation::Context.new(client: client, session: session),
             )
         end
 

data/lib/mongo/collection/view/explainable.rb

@@ -53,7 +53,7 @@ module Mongo
         #
         # @return [ Hash ] A single document with the query plan.
         #
-        # @see https://docs.mongodb.com/manual/reference/method/db.collection.explain/#db.collection.explain
+        # @see https://mongodb.com/docs/manual/reference/method/db.collection.explain/#db.collection.explain
         #
         # @since 2.0.0
         def explain(**opts)

data/lib/mongo/collection/view/iterable.rb

@@ -149,7 +149,7 @@ module Mongo
           }
         end
 
-        def initial_query_op(server, session)
+        def initial_query_op(session)
           spec = {
             coll_name: collection.name,
             filter: filter,
@@ -194,7 +194,7 @@ module Mongo
         end
 
         def send_initial_query(server, session = nil)
-          initial_query_op(server, session).execute(server, context: Operation::Context.new(client: client, session: session))
+          initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
 
         def use_query_cache?

data/lib/mongo/collection/view/map_reduce.rb

@@ -72,7 +72,7 @@ module Mongo
           @cursor = nil
           session = client.send(:get_session, @options)
           server = cluster.next_primary(nil, session)
-          result = send_initial_query(server, session)
+          result = send_initial_query(server, session, context: Operation::Context.new(client: client, session: session))
           result = send_fetch_query(server, session) unless inline?
           @cursor = Cursor.new(view, result, server, session: session)
           if block_given?
@@ -223,8 +223,9 @@ module Mongo
         def execute
           view.send(:with_session, @options) do |session|
             write_concern = view.write_concern_with_session(session)
-            nro_write_with_retry(session, write_concern) do |server|
-              send_initial_query(server, session)
+            context = Operation::Context.new(client: client, session: session)
+            nro_write_with_retry(write_concern, context: context) do |connection, txn_num, context|
+              send_initial_query_with_connection(connection, session, context: context)
             end
           end
         end
@@ -266,11 +267,10 @@ module Mongo
           Operation::MapReduce.new(spec)
         end
 
-        def valid_server?(server)
+        def valid_server?(description)
           if secondary_ok?
             true
           else
-            description = server.description
             description.standalone? || description.mongos? || description.primary? || description.load_balancer?
           end
         end
@@ -279,13 +279,22 @@ module Mongo
           out.respond_to?(:keys) && out.keys.first.to_s.downcase == INLINE
         end
 
-        def send_initial_query(server, session)
-          unless valid_server?(server)
-            msg = "Rerouting the MapReduce operation to the primary server - #{server.summary} is not suitable"
+        def send_initial_query(server, session, context:)
+          server.with_connection do |connection|
+            send_initial_query_with_connection(connection, session, context: context)
+          end
+        end
+
+        def send_initial_query_with_connection(connection, session, context:)
+          op = initial_query_op(session)
+          if valid_server?(connection.description)
+            op.execute_with_connection(connection, context: context)
+          else
+            msg = "Rerouting the MapReduce operation to the primary server - #{connection.address} is not suitable because it is not currently the primray"
             log_warn(msg)
             server = cluster.next_primary(nil, session)
+            op.execute(server, context: context)
           end
-          initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
 
         def fetch_query_spec

data/lib/mongo/collection/view/readable.rb

@@ -41,7 +41,8 @@ module Mongo
         # @option options [ true, false ] :bypass_document_validation Whether or
         #   not to skip document level validation.
         # @option options [ Hash ] :collation The collation to use.
-        # @option options [ String ] :comment Associate a comment with the aggregation.
+        # @option options [ Object ] :comment A user-provided
+        #   comment to attach to this command.
         # @option options [ String ] :hint The index to use for the aggregation.
         # @option options [ Hash ] :let Mapping of variables to use in the pipeline.
         #   See the server documentation for details.
@@ -57,6 +58,7 @@ module Mongo
         #
         # @since 2.0.0
         def aggregate(pipeline, options = {})
+          options = @options.merge(options) unless Mongo.broken_view_options
           aggregation = Aggregation.new(self, pipeline, options)
 
           # Because the $merge and $out pipeline stages write documents to the
@@ -127,7 +129,7 @@ module Mongo
         # @note Set profilingLevel to 2 and the comment will be logged in the profile
         #   collection along with the query.
         #
-        # @param [ String ] comment The comment to be associated with the query.
+        # @param [ Object ] comment The comment to be associated with the query.
         #
         # @return [ String, View ] Either the comment or a
         #   new +View+.
@@ -153,6 +155,8 @@ module Mongo
         # @option opts [ Hash ] :read The read preference options.
         # @option opts [ Hash ] :collation The collation to use.
         # @option opts [ Mongo::Session ] :session The session to use for the operation.
+        # @option opts [ Object ] :comment A user-provided
+        #   comment to attach to this command.
         #
         # @return [ Integer ] The document count.
         #
@@ -164,6 +168,7 @@ module Mongo
         #     * $near should be replaced with $geoWithin with $center
         #     * $nearSphere should be replaced with $geoWithin with $centerSphere
         def count(opts = {})
+          opts = @options.merge(opts) unless Mongo.broken_view_options
           cmd = { :count => collection.name, :query => filter }
           cmd[:skip] = opts[:skip] if opts[:skip]
           cmd[:hint] = opts[:hint] if opts[:hint]
@@ -187,6 +192,7 @@ module Mongo
                 # For some reason collation was historically accepted as a
                 # string key. Note that this isn't documented as valid usage.
                 collation: opts[:collation] || opts['collation'] || collation,
+                comment: opts[:comment],
               ).execute(server, context: Operation::Context.new(client: client, session: session))
             end.n.to_i
           end
@@ -207,17 +213,21 @@ module Mongo
         #   command to run.
         # @option opts [ Hash ] :read The read preference options.
         # @option opts [ Hash ] :collation The collation to use.
+        # @option opts [ Mongo::Session ] :session The session to use for the operation.
+        # @option ops [ Object ] :comment A user-provided
+        #   comment to attach to this command.
         #
         # @return [ Integer ] The document count.
         #
         # @since 2.6.0
         def count_documents(opts = {})
+          opts = @options.merge(opts) unless Mongo.broken_view_options
           pipeline = [:'$match' => filter]
           pipeline << { :'$skip' => opts[:skip] } if opts[:skip]
           pipeline << { :'$limit' => opts[:limit] } if opts[:limit]
           pipeline << { :'$group' => { _id: 1, n: { :'$sum' => 1 } } }
 
-          opts = opts.select { |k, _| [:hint, :max_time_ms, :read, :collation, :session].include?(k) }
+          opts = opts.slice(:hint, :max_time_ms, :read, :collation, :session, :comment)
           opts[:collation] ||= collation
 
           first = aggregate(pipeline, opts).first
@@ -235,6 +245,8 @@ module Mongo
         # @option opts :max_time_ms [ Integer ] The maximum amount of time to allow the command to
         #   run.
         # @option opts [ Hash ] :read The read preference options.
+        # @option opts [ Object ] :comment A user-provided
+        #   comment to attach to this command.
         #
         # @return [ Integer ] The document count.
         #
@@ -245,46 +257,34 @@ module Mongo
           end
 
           %i[limit skip].each do |opt|
-            if @options.key?(opt)
+            if options.key?(opt) || opts.key?(opt)
               raise ArgumentError, "Cannot call estimated_document_count when querying with #{opt}"
             end
           end
 
+          opts = @options.merge(opts) unless Mongo.broken_view_options
           Mongo::Lint.validate_underscore_read_preference(opts[:read])
           read_pref = opts[:read] || read_preference
           selector = ServerSelector.get(read_pref || server_selector)
           with_session(opts) do |session|
             read_with_retry(session, selector) do |server|
               context = Operation::Context.new(client: client, session: session)
-              if server.description.server_version_gte?('5.0')
-                pipeline = [
-                  {'$collStats' => {'count' => {}}},
-                  {'$group' => {'_id' => 1, 'n' => {'$sum' => '$count'}}},
-                ]
-                spec = Builder::Aggregation.new(
-                  pipeline,
-                  self,
-                  options.merge(session: session)
-                ).specification
-                result = Operation::Aggregate.new(spec).execute(server, context: context)
-                result.documents.first.fetch('n')
-              else
-                cmd = { count: collection.name }
-                cmd[:maxTimeMS] = opts[:max_time_ms] if opts[:max_time_ms]
-                if read_concern
-                  cmd[:readConcern] = Options::Mapper.transform_values_to_strings(
-                    read_concern)
+              cmd = { count: collection.name }
+              cmd[:maxTimeMS] = opts[:max_time_ms] if opts[:max_time_ms]
+              if read_concern
+                cmd[:readConcern] = Options::Mapper.transform_values_to_strings(
+                  read_concern)
                 end
                 result = Operation::Count.new(
                   selector: cmd,
                   db_name: database.name,
                   read: read_pref,
                   session: session,
+                  comment: opts[:comment],
                 ).execute(server, context: context)
                 result.n.to_i
               end
             end
-          end
         rescue Error::OperationFailure => exc
           if exc.code == 26
             # NamespaceNotFound
@@ -309,6 +309,8 @@ module Mongo
         #   command to run.
         # @option opts [ Hash ] :read The read preference options.
         # @option opts [ Hash ] :collation The collation to use.
+        # @option options [ Object ] :comment A user-provided
+        #   comment to attach to this command.
         #
         # @return [ Array<Object> ] The list of distinct values.
         #
@@ -317,9 +319,10 @@ module Mongo
           if field_name.nil?
             raise ArgumentError, 'Field name for distinct operation must be not nil'
           end
+          opts = @options.merge(opts) unless Mongo.broken_view_options
           cmd = { :distinct => collection.name,
                   :key => field_name.to_s,
-                  :query => filter }
+                  :query => filter, }
           cmd[:maxTimeMS] = opts[:max_time_ms] if opts[:max_time_ms]
           if read_concern
             cmd[:readConcern] = Options::Mapper.transform_values_to_strings(
@@ -336,6 +339,7 @@ module Mongo
                 options: {:limit => -1},
                 read: read_pref,
                 session: session,
+                comment: opts[:comment],
                 # For some reason collation was historically accepted as a
                 # string key. Note that this isn't documented as valid usage.
                 collation: opts[:collation] || opts['collation'] || collation,
@@ -702,7 +706,7 @@ module Mongo
             context = Operation::Context.new(
               client: client,
               session: session,
-              service_id: result.connection_description.service_id,
+              connection_global_id: result.connection_global_id,
             )
             result = op.execute(server, context: context)
             Cursor.new(self, result, server, session: session)