mongo 2.14.0 → 2.16.0

data/lib/mongo/collection/view/builder/map_reduce.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,7 +36,7 @@ module Mongo
             scope: 'scope',
             verbose: 'verbose',
             bypass_document_validation: 'bypassDocumentValidation',
-            collation: 'collation'
+            collation: 'collation',
           ).freeze
 
           def_delegators :@view, :collection, :database, :filter, :read, :write_concern
@@ -68,36 +71,6 @@ module Mongo
             @options = options
           end
 
-          # Get the specification for issuing a find command on the map/reduce
-          # results.
-          #
-          # @example Get the command specification.
-          #   builder.command_specification
-          #
-          # @return [ Hash ] The specification.
-          #
-          # @since 2.2.0
-          def command_specification
-            {
-              selector: find_command,
-              db_name: query_database,
-              read: read,
-              session: options[:session]
-            }
-          end
-
-          # Get the specification for the document query after a map/reduce.
-          #
-          # @example Get the query specification.
-          #   builder.query_specification
-          #
-          # @return [ Hash ] The specification.
-          #
-          # @since 2.2.0
-          def query_specification
-            { selector: {}, options: {}, db_name: query_database, coll_name: query_collection }
-          end
-
           # Get the specification to pass to the map/reduce operation.
           #
           # @example Get the specification.
@@ -110,6 +83,8 @@ module Mongo
             spec = {
               selector: map_reduce_command,
               db_name: database.name,
+              # Note that selector just above may also have a read preference
+              # specified, per the #map_reduce_command method below.
               read: read,
               session: options[:session]
             }
@@ -118,8 +93,6 @@ module Mongo
 
           private
 
-          OUT_ACTIONS = [ :replace, :merge, :reduce ].freeze
-
           def write?(spec)
             if out = spec[:selector][:out]
               out.is_a?(String) ||
@@ -127,37 +100,30 @@ module Mongo
             end
           end
 
-          def find_command
-            BSON::Document.new('find' => query_collection, 'filter' => {})
-          end
-
           def map_reduce_command
             command = BSON::Document.new(
               :mapReduce => collection.name,
               :map => map,
               :reduce => reduce,
               :query => filter,
-              :out => { inline: 1 }
+              :out => { inline: 1 },
             )
+            # Shouldn't this use self.read ?
             if collection.read_concern
               command[:readConcern] = Options::Mapper.transform_values_to_strings(
                 collection.read_concern)
             end
-            command.merge!(view_options)
+            command.update(view_options)
+            command.update(Utils.slice_hash(options, :collation))
+            # Read preference isn't simply passed in the command payload
+            # (it may need to be converted to wire protocol flags)
+            # so remove it here and hopefully it's handled elsewhere.
+            # If not, RUBY-2706.
+            command.delete(:read)
             command.merge!(Options::Mapper.transform_documents(options, MAPPINGS))
             command
           end
 
-          def query_database
-            options[:out].respond_to?(:keys) && options[:out][:db] ? options[:out][:db] : database.name
-          end
-
-          def query_collection
-            if options[:out].respond_to?(:keys)
-              options[:out][OUT_ACTIONS.find { |action| options[:out][action] }]
-            end || options[:out]
-          end
-
           def view_options
             @view_options ||= (opts = view.options.dup
                                opts.delete(:session)

data/lib/mongo/collection/view/builder.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,7 +17,3 @@
 
 require 'mongo/collection/view/builder/aggregation'
 require 'mongo/collection/view/builder/map_reduce'
-require 'mongo/collection/view/builder/op_query'
-require 'mongo/collection/view/builder/find_command'
-require 'mongo/collection/view/builder/flags'
-require 'mongo/collection/view/builder/modifiers'

data/lib/mongo/collection/view/change_stream/retryable.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2017-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/collection/view/change_stream.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2017-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -203,7 +206,7 @@ module Mongo
           unless closed?
             begin
               @cursor.close
-            rescue Error::OperationFailure
+            rescue Error::OperationFailure, Error::SocketError, Error::SocketTimeoutError
               # ignore
             end
             @cursor = nil
@@ -346,7 +349,7 @@ module Mongo
         end
 
         def send_initial_query(server, session)
-          initial_query_op(session).execute(server, client: client)
+          initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
 
         def time_to_bson_timestamp(time)

data/lib/mongo/collection/view/explainable.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/collection/view/immutable.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

data/lib/mongo/collection/view/iterable.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,6 +25,13 @@ module Mongo
       # @since 2.0.0
       module Iterable
 
+        # Returns the cursor associated with this view, if any.
+        #
+        # @return [ nil | Cursor ] The cursor, if any.
+        #
+        # @api private
+        attr_reader :cursor
+
         # Iterate through documents returned by a query with this +View+.
         #
         # @example Iterate through the result of the view.
@@ -35,18 +45,28 @@ module Mongo
         #
         # @yieldparam [ Hash ] Each matching document.
         def each
-          @cursor = if use_query_cache? && cached_cursor
+          # If the caching cursor is closed and was not fully iterated,
+          # the documents we have in it are not the complete result set and
+          # we have no way of completing that iteration.
+          # Therefore, discard that cursor and start iteration again.
+          # The case of the caching cursor not being closed and not having
+          # been fully iterated isn't tested - see RUBY-2773.
+          @cursor = if use_query_cache? && cached_cursor && (
+            cached_cursor.fully_iterated? || !cached_cursor.closed?
+          )
             cached_cursor
           else
             session = client.send(:get_session, @options)
-            select_cursor(session)
+            select_cursor(session).tap do |cursor|
+              if use_query_cache?
+                # No need to store the cursor in the query cache if there is
+                # already a cached cursor stored at this key.
+                QueryCache.set(cursor, **cache_options)
+              end
+            end
           end
 
           if use_query_cache?
-            # No need to store the cursor in the query cache if there is
-            # already a cached cursor stored at this key.
-            QueryCache.set(@cursor, **cache_options) unless cached_cursor
-
             # If a query with a limit is performed, the query cache will
             # re-use results from an earlier query with the same or larger
             # limit, and then impose the lower limit during iteration.
@@ -127,37 +147,54 @@ module Mongo
             projection: projection,
             collation: collation,
             read_concern: read_concern,
-            read_preference: read_preference
-
+            read_preference: read_preference,
           }
         end
 
         def initial_query_op(server, session)
-          if server.with_connection { |connection| connection.features }.find_command_enabled?
-            initial_command_op(session)
-          else
-            # Server versions that do not have the find command feature
-            # (versions older than 3.2) do not support the allow_disk_use option
-            # but perform no validation and will not raise an error if it is
-            # specified. If the allow_disk_use option is specified, raise an error
-            # to alert the user.
-            raise Error::UnsupportedOption.allow_disk_use_error if options.key?(:allow_disk_use)
-            Operation::Find.new(Builder::OpQuery.new(self).specification)
+          spec = {
+            coll_name: collection.name,
+            filter: filter,
+            projection: projection,
+            db_name: database.name,
+            session: session,
+            collation: collation,
+            sort: sort,
+            skip: skip,
+            limit: limit,
+            allow_disk_use: options[:allow_disk_use],
+            read: read,
+            read_concern: options[:read_concern] || read_concern,
+            batch_size: batch_size,
+            hint: options[:hint],
+            max_scan: options[:max_scan],
+            max_time_ms: options[:max_time_ms],
+            max_value: options[:max_value],
+            min_value: options[:min_value],
+            return_key: options[:return_key],
+            show_disk_loc: options[:show_disk_loc],
+            comment: options[:comment],
+            oplog_replay: if (v = options[:oplog_replay]).nil?
+              collection.options[:oplog_replay]
+            else
+              v
+            end,
+          }
+
+          if spec[:oplog_replay]
+            collection.client.log_warn("The :oplog_replay option is deprecated and ignored by MongoDB 4.4 and later")
           end
-        end
 
-        def initial_command_op(session)
-          builder = Builder::FindCommand.new(self, session)
           if explained?
-            Operation::Explain.new(builder.explain_specification)
+            spec[:explain] = options[:explain]
+            Operation::Explain.new(spec)
           else
-            Operation::Find.new(builder.specification)
+            Operation::Find.new(spec)
           end
         end
 
         def send_initial_query(server, session = nil)
-          validate_collation!(server, collation)
-          initial_query_op(server, session).execute(server, client: client)
+          initial_query_op(server, session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
 
         def use_query_cache?

data/lib/mongo/collection/view/map_reduce.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -153,6 +156,28 @@ module Mongo
           configure(:out, location)
         end
 
+        # Returns the collection name where the map-reduce result is written to.
+        # If the result is returned inline, returns nil.
+        def out_collection_name
+          if options[:out].respond_to?(:keys)
+            options[:out][OUT_ACTIONS.find do |action|
+              options[:out][action]
+            end]
+          end || options[:out]
+        end
+
+        # Returns the database name where the map-reduce result is written to.
+        # If the result is returned inline, returns nil.
+        def out_database_name
+          if options[:out]
+            if options[:out].respond_to?(:keys) && (db = options[:out][:db])
+              db
+            else
+              database.name
+            end
+          end
+        end
+
         # Set or get a scope on the operation.
         #
         # @example Set the scope value.
@@ -204,6 +229,8 @@ module Mongo
 
         private
 
+        OUT_ACTIONS = [ :replace, :merge, :reduce ].freeze
+
         def server_selector
           @view.send(:server_selector)
         end
@@ -225,10 +252,12 @@ module Mongo
         end
 
         def valid_server?(server)
-          description = server.with_connection do |connection|
-            connection.description
+          if secondary_ok?
+            true
+          else
+            description = server.description
+            description.standalone? || description.mongos? || description.primary? || description.load_balancer?
           end
-          description.standalone? || description.mongos? || description.primary? || secondary_ok?
         end
 
         def secondary_ok?
@@ -241,8 +270,7 @@ module Mongo
             log_warn(msg)
             server = cluster.next_primary(nil, session)
           end
-          validate_collation!(server)
-          initial_query_op(session).execute(server, client: client)
+          initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
 
         def fetch_query_spec
@@ -254,21 +282,20 @@ module Mongo
         end
 
         def fetch_query_op(server, session)
-          if server.with_connection { |connection| connection.features }.find_command_enabled?
-            Operation::Find.new(find_command_spec(session))
-          else
-            Operation::Find.new(fetch_query_spec)
-          end
+          spec = {
+            coll_name: out_collection_name,
+            db_name: out_database_name,
+            filter: {},
+            session: session,
+            read: read,
+            read_concern: options[:read_concern] || collection.read_concern,
+            collation: options[:collation] || view.options[:collation],
+          }
+          Operation::Find.new(spec)
         end
 
         def send_fetch_query(server, session)
-          fetch_query_op(server, session).execute(server, client: client)
-        end
-
-        def validate_collation!(server)
-          if (view.options[:collation] || options[:collation]) && !server.with_connection { |connection| connection.features }.collation_enabled?
-            raise Error::UnsupportedCollation.new
-          end
+          fetch_query_op(server, session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
       end
     end

data/lib/mongo/collection/view/readable.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,16 +24,6 @@ module Mongo
       # @since 2.0.0
       module Readable
 
-        # The query modifier constant.
-        #
-        # @since 2.2.0
-        QUERY = '$query'.freeze
-
-        # The modifiers option constant.
-        #
-        # @since 2.2.0
-        MODIFIERS = 'modifiers'.freeze
-
         # Execute an aggregation on the collection view.
         #
         # @example Aggregate documents.
@@ -166,14 +159,16 @@ module Mongo
           selector = ServerSelector.get(read_pref || server_selector)
           with_session(opts) do |session|
             read_with_retry(session, selector) do |server|
-              apply_collation!(cmd, server, opts)
               Operation::Count.new(
-                                     :selector => cmd,
-                                     :db_name => database.name,
-                                     :options => {:limit => -1},
-                                     :read => read_pref,
-                                     :session => session
-              ).execute(server, client: client)
+                selector: cmd,
+                db_name: database.name,
+                options: {:limit => -1},
+                read: read_pref,
+                session: session,
+                # For some reason collation was historically accepted as a
+                # string key. Note that this isn't documented as valid usage.
+                collation: opts[:collation] || opts['collation'] || collation,
+              ).execute(server, context: Operation::Context.new(client: client, session: session))
             end.n.to_i
           end
         end
@@ -230,24 +225,52 @@ module Mongo
             raise ArgumentError, "Cannot call estimated_document_count when querying with a filter"
           end
 
-          cmd = { count: collection.name }
-          cmd[:maxTimeMS] = opts[:max_time_ms] if opts[:max_time_ms]
-          if read_concern
-            cmd[:readConcern] = Options::Mapper.transform_values_to_strings(
-              read_concern)
+          %i[limit skip].each do |opt|
+            if @options.key?(opt)
+              raise ArgumentError, "Cannot call estimated_document_count when querying with #{opt}"
+            end
           end
+
           Mongo::Lint.validate_underscore_read_preference(opts[:read])
           read_pref = opts[:read] || read_preference
           selector = ServerSelector.get(read_pref || server_selector)
           with_session(opts) do |session|
             read_with_retry(session, selector) do |server|
-              Operation::Count.new(
-                selector: cmd,
-                db_name: database.name,
-                read: read_pref,
-                session: session
-              ).execute(server, client: client)
-            end.n.to_i
+              context = Operation::Context.new(client: client, session: session)
+              if server.description.server_version_gte?('5.0')
+                pipeline = [
+                  {'$collStats' => {'count' => {}}},
+                  {'$group' => {'_id' => 1, 'n' => {'$sum' => '$count'}}},
+                ]
+                spec = Builder::Aggregation.new(pipeline, self, options.merge(session: session)).specification
+                result = Operation::Aggregate.new(spec).execute(server, context: context)
+                result.documents.first.fetch('n')
+              else
+                cmd = { count: collection.name }
+                cmd[:maxTimeMS] = opts[:max_time_ms] if opts[:max_time_ms]
+                if read_concern
+                  cmd[:readConcern] = Options::Mapper.transform_values_to_strings(
+                    read_concern)
+                end
+                result = Operation::Count.new(
+                  selector: cmd,
+                  db_name: database.name,
+                  read: read_pref,
+                  session: session,
+                ).execute(server, context: context)
+                result.n.to_i
+              end
+            end
+          end
+        rescue Error::OperationFailure => exc
+          if exc.code == 26
+            # NamespaceNotFound
+            # This should only happen with the aggregation pipeline path
+            # (server 4.9+). Previous servers should return 0 for nonexistent
+            # collections.
+            0
+          else
+            raise
           end
         end
 
@@ -284,14 +307,16 @@ module Mongo
           selector = ServerSelector.get(read_pref || server_selector)
           with_session(opts) do |session|
             read_with_retry(session, selector) do |server|
-              apply_collation!(cmd, server, opts)
-              Operation::Distinct.new({
-                                        :selector => cmd,
-                                        :db_name => database.name,
-                                        :options => {:limit => -1},
-                                        :read => read_pref,
-                                        :session => session
-                                       }).execute(server, client: client)
+              Operation::Distinct.new(
+                selector: cmd,
+                db_name: database.name,
+                options: {:limit => -1},
+                read: read_pref,
+                session: session,
+                # For some reason collation was historically accepted as a
+                # string key. Note that this isn't documented as valid usage.
+                collation: opts[:collation] || opts['collation'] || collation,
+              ).execute(server, context: Operation::Context.new(client: client, session: session))
             end.first['values']
           end
         end
@@ -511,7 +536,11 @@ module Mongo
           configure(:sort, spec)
         end
 
-        # “meta” operators that let you modify the output or behavior of a query.
+        # If called without arguments or with a nil argument, returns
+        # the legacy (OP_QUERY) server modifiers for the current view.
+        # If called with a non-nil argument, which must be a Hash or a
+        # subclass, merges the provided modifiers into the current view.
+        # Both string and symbol keys are allowed in the input hash.
         #
         # @example Set the modifiers document.
         #   view.modifiers(:$orderby => Mongo::Index::ASCENDING)
@@ -522,8 +551,11 @@ module Mongo
         #
         # @since 2.1.0
         def modifiers(doc = nil)
-          return Builder::Modifiers.map_server_modifiers(options) if doc.nil?
-          new(options.merge(Builder::Modifiers.map_driver_options(doc)))
+          if doc.nil?
+            Operation::Find::Builder::Modifiers.map_server_modifiers(options)
+          else
+            new(options.merge(Operation::Find::Builder::Modifiers.map_driver_options(BSON::Document.new(doc))))
+          end
         end
 
         # A cumulative time limit in milliseconds for processing get more operations
@@ -605,43 +637,51 @@ module Mongo
         end
 
         def server_selector
-          if options[:session] && options[:session].in_transaction?
+          @server_selector ||= if options[:session] && options[:session].in_transaction?
             ServerSelector.get(read_preference || client.server_selector)
           else
-            @server_selector ||= ServerSelector.get(read_preference || collection.server_selector)
+            ServerSelector.get(read_preference || collection.server_selector)
           end
         end
 
         def parallel_scan(cursor_count, options = {})
           if options[:session]
+            # The session would be overwritten by the one in +options+ later.
             session = client.send(:get_session, @options)
           else
             session = nil
           end
           server = server_selector.select_server(cluster, nil, session)
-          cmd = Operation::ParallelScan.new({
-                  :coll_name => collection.name,
-                  :db_name => database.name,
-                  :cursor_count => cursor_count,
-                  :read_concern => read_concern,
-                  :session => session,
-                }.merge!(options))
-          cmd.execute(server, client: client).cursor_ids.map do |cursor_id|
-            result = if server.with_connection { |connection| connection.features }.find_command_enabled?
-              Operation::GetMore.new({
-                :selector => {:getMore => BSON::Int64.new(cursor_id),
-                             :collection => collection.name},
-                :db_name => database.name,
-                :session => session,
-              }).execute(server, client: client)
-             else
-              Operation::GetMore.new({
-                :to_return => 0,
-                :cursor_id => BSON::Int64.new(cursor_id),
-                :db_name => database.name,
-                :coll_name => collection.name
-              }).execute(server, client: client)
-            end
+          spec = {
+            coll_name: collection.name,
+            db_name: database.name,
+            cursor_count: cursor_count,
+            read_concern: read_concern,
+            session: session,
+          }.update(options)
+          session = spec[:session]
+          op = Operation::ParallelScan.new(spec)
+          # Note that the context object shouldn't be reused for subsequent
+          # GetMore operations.
+          context = Operation::Context.new(client: client, session: session)
+          result = op.execute(server, context: context)
+          result.cursor_ids.map do |cursor_id|
+            spec = {
+              cursor_id: cursor_id,
+              coll_name: collection.name,
+              db_name: database.name,
+              session: session,
+              batch_size: batch_size,
+              to_return: 0,
+              # max_time_ms is not being passed here, I assume intentionally?
+            }
+            op = Operation::GetMore.new(spec)
+            context = Operation::Context.new(
+              client: client,
+              session: session,
+              service_id: result.connection_description.service_id,
+            )
+            result = op.execute(server, context: context)
             Cursor.new(self, result, server, session: session)
           end
         end