mongo 2.15.0 → 2.18.1

data/lib/mongo/collection/view/builder/aggregation.rb

@@ -30,16 +30,17 @@ module Mongo
           #
           # @since 2.2.0
           MAPPINGS = BSON::Document.new(
-            :allow_disk_use => 'allowDiskUse',
-            :max_time_ms => 'maxTimeMS',
+            allow_disk_use: 'allowDiskUse',
+            bypass_document_validation: 'bypassDocumentValidation',
+            explain: 'explain',
+            collation: 'collation',
+            comment: 'comment',
+            hint: 'hint',
+            let: 'let',
             # This is intentional; max_await_time_ms is an alias for maxTimeMS
             # used on getMore commands for change streams.
-            :max_await_time_ms => 'maxTimeMS',
-            :explain => 'explain',
-            :bypass_document_validation => 'bypassDocumentValidation',
-            :collation => 'collation',
-            :hint => 'hint',
-            :comment => 'comment'
+            max_await_time_ms: 'maxTimeMS',
+            max_time_ms: 'maxTimeMS',
           ).freeze
 
           def_delegators :@view, :collection, :database, :read, :write_concern
@@ -55,12 +56,9 @@ module Mongo
 
           # Initialize the builder.
           #
-          # @example Initialize the builder.
-          #   Aggregation.new(map, reduce, view, options)
-          #
           # @param [ Array<Hash> ] pipeline The aggregation pipeline.
           # @param [ Collection::View ] view The collection view.
-          # @param [ Hash ] options The map/reduce options.
+          # @param [ Hash ] options The map/reduce and read preference options.
           #
           # @since 2.2.0
           def initialize(pipeline, view, options)
@@ -79,11 +77,12 @@ module Mongo
           # @since 2.2.0
           def specification
             spec = {
-                    selector: aggregation_command,
-                    db_name: database.name,
-                    read: view.read_preference,
-                    session: @options[:session]
-                   }
+              selector: aggregation_command,
+              db_name: database.name,
+              read: @options[:read_preference] || view.read_preference,
+              session: @options[:session],
+              collation: @options[:collation],
+            }
             if write?
               spec.update(write_concern: write_concern)
             end

data/lib/mongo/collection/view/builder/map_reduce.rb

@@ -36,7 +36,7 @@ module Mongo
             scope: 'scope',
             verbose: 'verbose',
             bypass_document_validation: 'bypassDocumentValidation',
-            collation: 'collation'
+            collation: 'collation',
           ).freeze
 
           def_delegators :@view, :collection, :database, :filter, :read, :write_concern
@@ -71,36 +71,6 @@ module Mongo
             @options = options
           end
 
-          # Get the specification for issuing a find command on the map/reduce
-          # results.
-          #
-          # @example Get the command specification.
-          #   builder.command_specification
-          #
-          # @return [ Hash ] The specification.
-          #
-          # @since 2.2.0
-          def command_specification
-            {
-              selector: find_command,
-              db_name: query_database,
-              read: read,
-              session: options[:session]
-            }
-          end
-
-          # Get the specification for the document query after a map/reduce.
-          #
-          # @example Get the query specification.
-          #   builder.query_specification
-          #
-          # @return [ Hash ] The specification.
-          #
-          # @since 2.2.0
-          def query_specification
-            { selector: {}, options: {}, db_name: query_database, coll_name: query_collection }
-          end
-
           # Get the specification to pass to the map/reduce operation.
           #
           # @example Get the specification.
@@ -113,6 +83,8 @@ module Mongo
             spec = {
               selector: map_reduce_command,
               db_name: database.name,
+              # Note that selector just above may also have a read preference
+              # specified, per the #map_reduce_command method below.
               read: read,
               session: options[:session]
             }
@@ -121,8 +93,6 @@ module Mongo
 
           private
 
-          OUT_ACTIONS = [ :replace, :merge, :reduce ].freeze
-
           def write?(spec)
             if out = spec[:selector][:out]
               out.is_a?(String) ||
@@ -130,35 +100,31 @@ module Mongo
             end
           end
 
-          def find_command
-            BSON::Document.new('find' => query_collection, 'filter' => {})
-          end
-
           def map_reduce_command
             command = BSON::Document.new(
               :mapReduce => collection.name,
               :map => map,
               :reduce => reduce,
               :query => filter,
-              :out => { inline: 1 }
+              :out => { inline: 1 },
             )
+            # Shouldn't this use self.read ?
             if collection.read_concern
               command[:readConcern] = Options::Mapper.transform_values_to_strings(
                 collection.read_concern)
             end
-            command.merge!(view_options)
-            command.merge!(Options::Mapper.transform_documents(options, MAPPINGS))
-            command
-          end
+            command.update(view_options)
+            command.update(options.slice(:collation))
 
-          def query_database
-            options[:out].respond_to?(:keys) && options[:out][:db] ? options[:out][:db] : database.name
-          end
+            # Read preference isn't simply passed in the command payload
+            # (it may need to be converted to wire protocol flags).
+            # Ideally it should be removed here, however due to Mongoid 7
+            # using this method and requiring :read to be returned from it,
+            # we cannot do this just yet - see RUBY-2932.
+            #command.delete(:read)
 
-          def query_collection
-            if options[:out].respond_to?(:keys)
-              options[:out][OUT_ACTIONS.find { |action| options[:out][action] }]
-            end || options[:out]
+            command.merge!(Options::Mapper.transform_documents(options, MAPPINGS))
+            command
           end
 
           def view_options

data/lib/mongo/collection/view/builder.rb

@@ -17,7 +17,3 @@
 
 require 'mongo/collection/view/builder/aggregation'
 require 'mongo/collection/view/builder/map_reduce'
-require 'mongo/collection/view/builder/op_query'
-require 'mongo/collection/view/builder/find_command'
-require 'mongo/collection/view/builder/flags'
-require 'mongo/collection/view/builder/modifiers'

data/lib/mongo/collection/view/change_stream.rb

@@ -70,10 +70,35 @@ module Mongo
         # @param [ Array<Hash> ] pipeline The pipeline of operators to filter the change notifications.
         # @param [ Hash ] options The change stream options.
         #
-        # @option options [ String ] :full_document Allowed values: 'default', 'updateLookup'. Defaults to 'default'.
-        #   When set to 'updateLookup', the change notification for partial updates will include both a delta
-        #   describing the changes to the document, as well as a copy of the entire document that was changed
-        #   from some time after the change occurred.
+        # @option options [ String ] :full_document Allowed values: nil, 'default',
+        #   'updateLookup', 'whenAvailable', 'required'.
+        #
+        #   The default is to not send a value (i.e. nil), which is equivalent to
+        #   'default'. By default, the change notification for partial updates will
+        #   include a delta describing the changes to the document.
+        #
+        #   When set to 'updateLookup', the change notification for partial updates
+        #   will include both a delta describing the changes to the document as well
+        #   as a copy of the entire document that was changed from some time after
+        #   the change occurred.
+        #
+        #   When set to 'whenAvailable', configures the change stream to return the
+        #   post-image of the modified document for replace and update change events
+        #   if the post-image for this event is available.
+        #
+        #   When set to 'required', the same behavior as 'whenAvailable' except that
+        #   an error is raised if the post-image is not available.
+        # @option options [ String ] :full_document_before_change Allowed values: nil,
+        #   'whenAvailable', 'required', 'off'.
+        #
+        #   The default is to not send a value (i.e. nil), which is equivalent to 'off'.
+        #
+        #   When set to 'whenAvailable', configures the change stream to return the
+        #   pre-image of the modified document for replace, update, and delete change
+        #   events if it is available.
+        #
+        #   When set to 'required', the same behavior as 'whenAvailable' except that
+        #   an error is raised if the pre-image is not available.
         # @option options [ BSON::Document, Hash ] :resume_after Specifies the logical starting point for the
         #   new change stream.
         # @option options [ Integer ] :max_await_time_ms The maximum amount of time for the server to wait
@@ -88,6 +113,13 @@ module Mongo
         #   option takes a resume token and starts a new change stream returning the first
         #   notification after the token. This will allow users to watch collections that have been
         #   dropped and recreated or newly renamed collections without missing any notifications.
+        # @option options [ Object ] :comment A user-provided
+        #   comment to attach to this command.
+        # @option options [ Boolean ] :show_expanded_events Enables the server to
+        #   send the 'expanded' list of change stream events. The list of additional
+        #   events included with this flag set are: createIndexes, dropIndexes,
+        #   modify, create, shardCollection, reshardCollection,
+        #   refineCollectionShardKey.
         #
         #   The server will report an error if `startAfter` and `resumeAfter` are both specified.
         #
@@ -275,20 +307,22 @@ module Mongo
           start_at_operation_time = nil
           start_at_operation_time_supported = nil
           @cursor = read_with_retry_cursor(session, server_selector, view) do |server|
-            start_at_operation_time_supported = server.description.server_version_gte?('4.0')
+            server.with_connection do |connection|
+              start_at_operation_time_supported = connection.description.server_version_gte?('4.0')
 
-            result = send_initial_query(server, session)
-            if doc = result.replies.first && result.replies.first.documents.first
-              start_at_operation_time = doc['operationTime']
-            else
-              # The above may set @start_at_operation_time to nil
-              # if it was not in the document for some reason,
-              # for consistency set it to nil here as well.
-              # NB: since this block may be executed more than once, each
-              # execution must write to start_at_operation_time either way.
-              start_at_operation_time = nil
+              result = send_initial_query(connection, session)
+              if doc = result.replies.first && result.replies.first.documents.first
+                start_at_operation_time = doc['operationTime']
+              else
+                # The above may set @start_at_operation_time to nil
+                # if it was not in the document for some reason,
+                # for consistency set it to nil here as well.
+                # NB: since this block may be executed more than once, each
+                # execution must write to start_at_operation_time either way.
+                start_at_operation_time = nil
+              end
+              result
             end
-            result
           end
           @start_at_operation_time = start_at_operation_time
           @start_at_operation_time_supported = start_at_operation_time_supported
@@ -298,8 +332,8 @@ module Mongo
           [{ '$changeStream' => change_doc }] + @change_stream_filters
         end
 
-        def aggregate_spec(session)
-          super(session).tap do |spec|
+        def aggregate_spec(session, read_preference)
+          super(session, read_preference).tap do |spec|
             spec[:selector][:aggregate] = 1 unless for_collection?
           end
         end
@@ -310,6 +344,14 @@ module Mongo
               doc[:fullDocument] = @options[:full_document]
             end
 
+            if @options[:full_document_before_change]
+              doc[:fullDocumentBeforeChange] = @options[:full_document_before_change]
+            end
+
+            if @options.key?(:show_expanded_events)
+              doc[:showExpandedEvents] = @options[:show_expanded_events]
+            end
+
             if resuming?
               # We have a resume token once we retrieved any documents.
               # However, if the first getMore fails and the user didn't pass
@@ -348,8 +390,12 @@ module Mongo
           end
         end
 
-        def send_initial_query(server, session)
-          initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session))
+        def send_initial_query(connection, session)
+          initial_query_op(session, view.read_preference)
+            .execute_with_connection(
+              connection,
+              context: Operation::Context.new(client: client, session: session),
+            )
         end
 
         def time_to_bson_timestamp(time)

data/lib/mongo/collection/view/explainable.rb

@@ -53,7 +53,7 @@ module Mongo
         #
         # @return [ Hash ] A single document with the query plan.
         #
-        # @see https://docs.mongodb.com/manual/reference/method/db.collection.explain/#db.collection.explain
+        # @see https://mongodb.com/docs/manual/reference/method/db.collection.explain/#db.collection.explain
         #
         # @since 2.0.0
         def explain(**opts)

data/lib/mongo/collection/view/iterable.rb

@@ -25,6 +25,13 @@ module Mongo
       # @since 2.0.0
       module Iterable
 
+        # Returns the cursor associated with this view, if any.
+        #
+        # @return [ nil | Cursor ] The cursor, if any.
+        #
+        # @api private
+        attr_reader :cursor
+
         # Iterate through documents returned by a query with this +View+.
         #
         # @example Iterate through the result of the view.
@@ -38,22 +45,32 @@ module Mongo
         #
         # @yieldparam [ Hash ] Each matching document.
         def each
-          @cursor = if use_query_cache? && cached_cursor
+          # If the caching cursor is closed and was not fully iterated,
+          # the documents we have in it are not the complete result set and
+          # we have no way of completing that iteration.
+          # Therefore, discard that cursor and start iteration again.
+          # The case of the caching cursor not being closed and not having
+          # been fully iterated isn't tested - see RUBY-2773.
+          @cursor = if use_query_cache? && cached_cursor && (
+            cached_cursor.fully_iterated? || !cached_cursor.closed?
+          )
             cached_cursor
           else
             session = client.send(:get_session, @options)
-            select_cursor(session)
+            select_cursor(session).tap do |cursor|
+              if use_query_cache?
+                # No need to store the cursor in the query cache if there is
+                # already a cached cursor stored at this key.
+                QueryCache.set(cursor, **cache_options)
+              end
+            end
           end
 
           if use_query_cache?
-            # No need to store the cursor in the query cache if there is
-            # already a cached cursor stored at this key.
-            QueryCache.set(@cursor, **cache_options) unless cached_cursor
-
             # If a query with a limit is performed, the query cache will
             # re-use results from an earlier query with the same or larger
             # limit, and then impose the lower limit during iteration.
-            limit_for_cached_query = respond_to?(:limit) ? limit : nil
+            limit_for_cached_query = respond_to?(:limit) ? QueryCache.normalized_limit(limit) : nil
           end
 
           if block_given?
@@ -97,11 +114,9 @@ module Mongo
 
         def select_cursor(session)
           if respond_to?(:write?, true) && write?
-            server = server_selector.select_server(cluster, nil, session)
+            server = server_selector.select_server(cluster, nil, session, write_aggregation: true)
             result = send_initial_query(server, session)
 
-            # RUBY-2367: This will be updated to allow the query cache to
-            # cache cursors with multi-batch results.
             if use_query_cache?
               CachingCursor.new(view, result, server, session: session)
             else
@@ -130,37 +145,56 @@ module Mongo
             projection: projection,
             collation: collation,
             read_concern: read_concern,
-            read_preference: read_preference
-
+            read_preference: read_preference,
           }
         end
 
-        def initial_query_op(server, session)
-          if server.with_connection { |connection| connection.features }.find_command_enabled?
-            initial_command_op(session)
-          else
-            # Server versions that do not have the find command feature
-            # (versions older than 3.2) do not support the allow_disk_use option
-            # but perform no validation and will not raise an error if it is
-            # specified. If the allow_disk_use option is specified, raise an error
-            # to alert the user.
-            raise Error::UnsupportedOption.allow_disk_use_error if options.key?(:allow_disk_use)
-            Operation::Find.new(Builder::OpQuery.new(self).specification)
+        def initial_query_op(session)
+          spec = {
+            coll_name: collection.name,
+            filter: filter,
+            projection: projection,
+            db_name: database.name,
+            session: session,
+            collation: collation,
+            sort: sort,
+            skip: skip,
+            let: options[:let],
+            limit: limit,
+            allow_disk_use: options[:allow_disk_use],
+            read: read,
+            read_concern: options[:read_concern] || read_concern,
+            batch_size: batch_size,
+            hint: options[:hint],
+            max_scan: options[:max_scan],
+            max_time_ms: options[:max_time_ms],
+            max_value: options[:max_value],
+            min_value: options[:min_value],
+            no_cursor_timeout: options[:no_cursor_timeout],
+            return_key: options[:return_key],
+            show_disk_loc: options[:show_disk_loc],
+            comment: options[:comment],
+            oplog_replay: if (v = options[:oplog_replay]).nil?
+              collection.options[:oplog_replay]
+            else
+              v
+            end,
+          }
+
+          if spec[:oplog_replay]
+            collection.client.log_warn("The :oplog_replay option is deprecated and ignored by MongoDB 4.4 and later")
           end
-        end
 
-        def initial_command_op(session)
-          builder = Builder::FindCommand.new(self, session)
           if explained?
-            Operation::Explain.new(builder.explain_specification)
+            spec[:explain] = options[:explain]
+            Operation::Explain.new(spec)
           else
-            Operation::Find.new(builder.specification)
+            Operation::Find.new(spec)
           end
         end
 
         def send_initial_query(server, session = nil)
-          validate_collation!(server, collation)
-          initial_query_op(server, session).execute(server, context: Operation::Context.new(client: client, session: session))
+          initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
 
         def use_query_cache?

data/lib/mongo/collection/view/map_reduce.rb

@@ -72,7 +72,7 @@ module Mongo
           @cursor = nil
           session = client.send(:get_session, @options)
           server = cluster.next_primary(nil, session)
-          result = send_initial_query(server, session)
+          result = send_initial_query(server, session, context: Operation::Context.new(client: client, session: session))
           result = send_fetch_query(server, session) unless inline?
           @cursor = Cursor.new(view, result, server, session: session)
           if block_given?
@@ -115,6 +115,8 @@ module Mongo
           @map_function = map.dup.freeze
           @reduce_function = reduce.dup.freeze
           @options = BSON::Document.new(options).freeze
+
+          client.log_warn('The map_reduce operation is deprecated, please use the aggregation pipeline instead')
         end
 
         # Set or get the jsMode flag for the operation.
@@ -156,6 +158,28 @@ module Mongo
           configure(:out, location)
         end
 
+        # Returns the collection name where the map-reduce result is written to.
+        # If the result is returned inline, returns nil.
+        def out_collection_name
+          if options[:out].respond_to?(:keys)
+            options[:out][OUT_ACTIONS.find do |action|
+              options[:out][action]
+            end]
+          end || options[:out]
+        end
+
+        # Returns the database name where the map-reduce result is written to.
+        # If the result is returned inline, returns nil.
+        def out_database_name
+          if options[:out]
+            if options[:out].respond_to?(:keys) && (db = options[:out][:db])
+              db
+            else
+              database.name
+            end
+          end
+        end
+
         # Set or get a scope on the operation.
         #
         # @example Set the scope value.
@@ -199,14 +223,17 @@ module Mongo
         def execute
           view.send(:with_session, @options) do |session|
             write_concern = view.write_concern_with_session(session)
-            nro_write_with_retry(session, write_concern) do |server|
-              send_initial_query(server, session)
+            context = Operation::Context.new(client: client, session: session)
+            nro_write_with_retry(write_concern, context: context) do |connection, txn_num, context|
+              send_initial_query_with_connection(connection, session, context: context)
             end
           end
         end
 
         private
 
+        OUT_ACTIONS = [ :replace, :merge, :reduce ].freeze
+
         def server_selector
           @view.send(:server_selector)
         end
@@ -224,28 +251,50 @@ module Mongo
         end
 
         def initial_query_op(session)
-          Operation::MapReduce.new(map_reduce_spec(session))
+          spec = map_reduce_spec(session)
+          # Read preference isn't simply passed in the command payload
+          # (it may need to be converted to wire protocol flags).
+          # Passing it in command payload produces errors on at least
+          # 5.0 mongoses.
+          # In the future map_reduce_command should remove :read
+          # from its return value, however we cannot do this right now
+          # due to Mongoid 7 relying on :read being returned as part of
+          # the command - see RUBY-2932.
+          # Delete :read here for now because it cannot be sent to mongos this way.
+          spec = spec.dup
+          spec[:selector] = spec[:selector].dup
+          spec[:selector].delete(:read)
+          Operation::MapReduce.new(spec)
         end
 
-        def valid_server?(server)
-          description = server.with_connection do |connection|
-            connection.description
+        def valid_server?(description)
+          if secondary_ok?
+            true
+          else
+            description.standalone? || description.mongos? || description.primary? || description.load_balancer?
           end
-          description.standalone? || description.mongos? || description.primary? || secondary_ok?
         end
 
         def secondary_ok?
           out.respond_to?(:keys) && out.keys.first.to_s.downcase == INLINE
         end
 
-        def send_initial_query(server, session)
-          unless valid_server?(server)
-            msg = "Rerouting the MapReduce operation to the primary server - #{server.summary} is not suitable"
+        def send_initial_query(server, session, context:)
+          server.with_connection do |connection|
+            send_initial_query_with_connection(connection, session, context: context)
+          end
+        end
+
+        def send_initial_query_with_connection(connection, session, context:)
+          op = initial_query_op(session)
+          if valid_server?(connection.description)
+            op.execute_with_connection(connection, context: context)
+          else
+            msg = "Rerouting the MapReduce operation to the primary server - #{connection.address} is not suitable because it is not currently the primray"
             log_warn(msg)
             server = cluster.next_primary(nil, session)
+            op.execute(server, context: context)
           end
-          validate_collation!(server)
-          initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
 
         def fetch_query_spec
@@ -257,22 +306,21 @@ module Mongo
         end
 
         def fetch_query_op(server, session)
-          if server.with_connection { |connection| connection.features }.find_command_enabled?
-            Operation::Find.new(find_command_spec(session))
-          else
-            Operation::Find.new(fetch_query_spec)
-          end
+          spec = {
+            coll_name: out_collection_name,
+            db_name: out_database_name,
+            filter: {},
+            session: session,
+            read: read,
+            read_concern: options[:read_concern] || collection.read_concern,
+            collation: options[:collation] || view.options[:collation],
+          }
+          Operation::Find.new(spec)
         end
 
         def send_fetch_query(server, session)
           fetch_query_op(server, session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
-
-        def validate_collation!(server)
-          if (view.options[:collation] || options[:collation]) && !server.with_connection { |connection| connection.features }.collation_enabled?
-            raise Error::UnsupportedCollation.new
-          end
-        end
       end
     end
   end