mongo 2.16.4 → 2.20.0

data/lib/mongo/cluster.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
@@ -157,7 +157,7 @@ module Mongo
       # @sdam_flow_lock covers just the sdam flow. Note it does not apply
       # to @topology replacements which are done under @update_lock.
       @sdam_flow_lock = Mutex.new
-      Session::SessionPool.create(self)
+      @session_pool = Session::SessionPool.new(self)
 
       if seeds.empty? && load_balanced?
         raise ArgumentError, 'Load-balanced clusters with no seeds are prohibited'
@@ -186,6 +186,8 @@ module Mongo
         recreate_topology(topology, opening_topology)
       end
 
+      possibly_warn_about_compatibility!
+
       if load_balanced?
         # We are required by the specifications to produce certain SDAM events
         # when in load-balanced topology.
@@ -263,10 +265,10 @@ module Mongo
             log_debug("Waiting for up to #{'%.2f' % time_remaining} seconds for servers to be scanned: #{summary}")
             # Since the semaphore may have been signaled between us checking
             # the servers list above and the wait call below, we should not
-            # wait for the full remaining time - wait for up to 1 second, then
+            # wait for the full remaining time - wait for up to 0.5 second, then
             # recheck the state.
             begin
-              server_selection_semaphore.wait([time_remaining, 1].min)
+              server_selection_semaphore.wait([time_remaining, 0.5].min)
             rescue ::Timeout::Error
               # nothing
             end
@@ -280,20 +282,22 @@ module Mongo
     # Create a cluster for the provided client, for use when we don't want the
     # client's original cluster instance to be the same.
     #
-    # @api private
-    #
     # @example Create a cluster for the client.
     #   Cluster.create(client)
     #
     # @param [ Client ] client The client to create on.
+    # @param [ Monitoring | nil ] monitoring. The monitoring instance to use
+    #   with the new cluster. If nil, a new instance of Monitoring will be
+    #   created.
     #
     # @return [ Cluster ] The cluster.
     #
     # @since 2.0.0
-    def self.create(client)
+    # @api private
+    def self.create(client, monitoring: nil)
       cluster = Cluster.new(
         client.cluster.addresses.map(&:to_s),
-        Monitoring.new,
+        monitoring || Monitoring.new,
         client.cluster_options,
       )
       client.instance_variable_set(:@cluster, cluster)
@@ -495,13 +499,16 @@ module Mongo
     # events in the process. Stops SRV monitoring if it is active.
     # Marks the cluster disconnected.
     #
-    # @return [ true ] Always true.
+    # A closed cluster is no longer usable. If the client is reconnected,
+    # it will create a new cluster instance.
     #
-    # @since 2.1.0
-    def disconnect!
+    # @return [ nil ] Always nil.
+    #
+    # @api private
+    def close
       @state_change_lock.synchronize do
         unless connecting? || connected?
-          return true
+          return nil
         end
         if options[:cleanup] != false
           session_pool.end_sessions
@@ -514,7 +521,7 @@ module Mongo
         end
         @servers.each do |server|
           if server.connected?
-            server.disconnect!
+            server.close
             publish_sdam_event(
               Monitoring::SERVER_CLOSED,
               Monitoring::Event::ServerClosed.new(server.address, topology)
@@ -529,7 +536,7 @@ module Mongo
           @connecting = @connected = false
         end
       end
-      true
+      nil
     end
 
     # Reconnect all servers.
@@ -620,10 +627,12 @@ module Mongo
     #   respective server is cleared. Set this option to true to keep the
     #   existing connection pool (required when handling not master errors
     #   on 4.2+ servers).
-    # @option aptions [ true | false ] :awaited Whether the updated description
+    # @option options [ true | false ] :awaited Whether the updated description
     #   was a result of processing an awaited hello.
     # @option options [ Object ] :service_id Change state for the specified
     #   service id only.
+    # @option options [ Mongo::Error | nil ] :scan_error The error encountered
+    #   while scanning, or nil if no error was raised.
     #
     # @api private
     def run_sdam_flow(previous_desc, updated_desc, options = {})
@@ -634,7 +643,9 @@ module Mongo
               # TODO should service id be taken out of updated_desc?
               # We could also assert that
               # options[:service_id] == updated_desc.service_id
-              server.clear_connection_pool(service_id: options[:service_id])
+              err = options[:scan_error]
+              interrupt = err && (err.is_a?(Error::SocketError) || err.is_a?(Error::SocketTimeoutError))
+              server.clear_connection_pool(service_id: options[:service_id], interrupt_in_use_connections: interrupt)
             end
           end
         end
@@ -654,7 +665,9 @@ module Mongo
           if flow.became_unknown?
             servers_list.each do |server|
               if server.address == updated_desc.address
-                server.clear_connection_pool
+                err = options[:scan_error]
+                interrupt = err && (err.is_a?(Error::SocketError) || err.is_a?(Error::SocketTimeoutError))
+                server.clear_connection_pool(interrupt_in_use_connections: interrupt)
               end
             end
           end
@@ -695,12 +708,16 @@ module Mongo
     # @api private
     def set_server_list(server_address_strs)
       @sdam_flow_lock.synchronize do
+        # If one of the new addresses is not in the current servers list,
+        # add it to the servers list.
         server_address_strs.each do |address_str|
           unless servers_list.any? { |server| server.address.seed == address_str }
             add(address_str)
           end
         end
 
+        # If one of the servers' addresses are not in the new address list,
+        # remove that server from the servers list.
         servers_list.each do |server|
           unless server_address_strs.any? { |address_str| server.address.seed == address_str }
             remove(server.address.seed)
@@ -822,6 +839,9 @@ module Mongo
       address = Address.new(host, options)
       if !addresses.include?(address)
         opts = options.merge(monitor: false)
+        # If we aren't starting the montoring threads, we also don't want to
+        # start the pool's populator thread.
+        opts.merge!(populator_io: false) unless options.fetch(:monitoring_io, true)
         # Note that in a load-balanced topology, every server must be a
         # load balancer (load_balancer: true is specified in the options)
         # but this option isn't set here because we are required by the
@@ -925,6 +945,7 @@ module Mongo
     # @api private
     def disconnect_server_if_connected(server)
       if server.connected?
+        server.clear_description
         server.disconnect!
         publish_sdam_event(
           Monitoring::SERVER_CLOSED,
@@ -937,11 +958,21 @@ module Mongo
     # is connected to does not support sessions.
     #
     # Session support may change over time, for example due to servers in the
-    # deployment being upgraded or downgraded. If the client is currently not
-    # connected to any data bearing servers, this method considers the state
-    # of session support as of when the client was last connected to at
-    # least one server. If the client has never connected to any servers,
-    # the deployment is considered to not support sessions.
+    # deployment being upgraded or downgraded.  If the client isn't connected to
+    # any servers and doesn't find any servers
+    # for the duration of server selection timeout, this method will raise
+    # NoServerAvailable. This method is called from the operation execution flow,
+    # and if it raises NoServerAvailable the entire operation will fail
+    # with that exception, since the operation execution has waited for
+    # the server selection timeout for any server to become available
+    # (which would be a superset of the servers suitable for the operation being
+    # attempted) and none materialized.
+    #
+    # @raise [ Error::SessionsNotAvailable ] If the deployment that the driver
+    #   is connected to does not support sessions.
+    # @raise [ Error::NoServerAvailable ] If the client isn't connected to
+    #   any servers and doesn't find any servers for the duration of
+    #   server selection timeout.
     #
     # @api private
     def validate_session_support!
@@ -962,25 +993,11 @@ module Mongo
       # No data bearing servers known - perform server selection to try to
       # get a response from at least one of them, to return an accurate
       # assessment of whether sessions are currently supported.
-      begin
-        ServerSelector.get(mode: :primary_preferred).select_server(self)
-        @state_change_lock.synchronize do
-          @sdam_flow_lock.synchronize do
-            unless topology.logical_session_timeout
-              raise_sessions_not_supported
-            end
-          end
-        end
-      rescue Error::NoServerAvailable
-        # We haven't been able to contact any servers - use last known
-        # value for session support.
-        @state_change_lock.synchronize do
-          @sdam_flow_lock.synchronize do
-            @update_lock.synchronize do
-              unless @sessions_supported
-                raise_sessions_not_supported
-              end
-            end
+      ServerSelector.get(mode: :primary_preferred).select_server(self)
+      @state_change_lock.synchronize do
+        @sdam_flow_lock.synchronize do
+          unless topology.logical_session_timeout
+            raise_sessions_not_supported
           end
         end
       end
@@ -1067,6 +1084,30 @@ module Mongo
         Monitoring::Event::TopologyChanged.new(previous_topology, @topology)
       )
     end
+
+    COSMOSDB_HOST_PATTERNS = %w[ .cosmos.azure.com ]
+    COSMOSDB_LOG_MESSAGE = 'You appear to be connected to a CosmosDB cluster. ' \
+      'For more information regarding feature compatibility and support please visit ' \
+      'https://www.mongodb.com/supportability/cosmosdb'
+
+    DOCUMENTDB_HOST_PATTERNS = %w[ .docdb.amazonaws.com .docdb-elastic.amazonaws.com ]
+    DOCUMENTDB_LOG_MESSAGE = 'You appear to be connected to a DocumentDB cluster. ' \
+      'For more information regarding feature compatibility and support please visit ' \
+      'https://www.mongodb.com/supportability/documentdb'
+
+    # Compares the server hosts with address suffixes of known services
+    # that provide limited MongoDB API compatibility, and warns about them.
+    def possibly_warn_about_compatibility!
+      if topology.server_hosts_match_any?(COSMOSDB_HOST_PATTERNS)
+        log_info COSMOSDB_LOG_MESSAGE
+        return
+      end
+
+      if topology.server_hosts_match_any?(DOCUMENTDB_HOST_PATTERNS)
+        log_info DOCUMENTDB_LOG_MESSAGE
+        return
+      end
+    end
   end
 end
 

data/lib/mongo/cluster_time.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2019-2020 MongoDB Inc.
 #

data/lib/mongo/collection/helpers.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+# rubocop:todo all
+
+# Copyright (C) 2014-2022 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Collection
+    # This module contains helper methods collection class.
+    #
+    # @api private
+    module Helpers
+      # Executes drop operation and and ignores NamespaceNotFound error.
+      #
+      # @param [ Operation::Drop ] operation Drop operation to be executed.
+      # @param [ Session ] session Session to be use for execution.
+      # @param [ Operation::Context ] context Context to use for execution.
+      #
+      # @return [ Result ] The result of the execution.
+      def do_drop(operation, session, context)
+        operation.execute(next_primary(nil, session), context: context)
+      rescue Error::OperationFailure => ex
+        # NamespaceNotFound
+        if ex.code == 26 || ex.code.nil? && ex.message =~ /ns not found/
+          false
+        else
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/mongo/collection/queryable_encryption.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+# Copyright (C) 2014-2022 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Collection
+    # This module contains methods for creating and dropping auxiliary collections
+    # for queryable encryption.
+    #
+    # @api private
+    module QueryableEncryption
+      # The minimum wire version for QE2 support
+      QE2_MIN_WIRE_VERSION = 21
+
+      # Creates auxiliary collections and indices for queryable encryption if necessary.
+      #
+      # @param [ Hash | nil ] encrypted_fields Encrypted fields hash that was
+      #   provided to `create` collection helper.
+      # @param [ Client ] client Mongo client to be used to create auxiliary collections.
+      # @param [ Session ] session Session to be used to create auxiliary collections.
+      #
+      # @return [ Result ] The result of provided block.
+      def maybe_create_qe_collections(encrypted_fields, client, session)
+        encrypted_fields = encrypted_fields_from(encrypted_fields)
+        return yield if encrypted_fields.empty?
+
+        server = next_primary(nil, session)
+        context = Operation::Context.new(client: client, session: session)
+        server.with_connection do |connection|
+          check_wire_version!(connection)
+          emm_collections(encrypted_fields).each do |coll|
+            create_operation_for(coll)
+              .execute_with_connection(connection, context: context)
+          end
+        end
+
+        yield(encrypted_fields).tap do |result|
+          indexes.create_one(__safeContent__: 1) if result
+        end
+      end
+
+      # Drops auxiliary collections and indices for queryable encryption if necessary.
+      #
+      # @param [ Hash | nil ] encrypted_fields Encrypted fields hash that was
+      #   provided to `create` collection helper.
+      # @param [ Client ] client Mongo client to be used to drop auxiliary collections.
+      # @param [ Session ] session Session to be used to drop auxiliary collections.
+      #
+      # @return [ Result ] The result of provided block.
+      def maybe_drop_emm_collections(encrypted_fields, client, session)
+        encrypted_fields = if encrypted_fields
+                             encrypted_fields
+                           elsif encrypted_fields_map
+                             encrypted_fields_for_drop_from_map
+                           else
+                             {}
+                           end
+
+        return yield if encrypted_fields.empty?
+
+        emm_collections(encrypted_fields).each do |coll|
+          context = Operation::Context.new(client: client, session: session)
+          operation = Operation::Drop.new(
+            selector: { drop: coll },
+            db_name: database.name,
+            session: session
+          )
+          do_drop(operation, session, context)
+        end
+
+        yield
+      end
+
+      private
+
+      # Checks if names for auxiliary collections are set and returns them,
+      # otherwise returns default names.
+      #
+      # @param [ Hash ] encrypted_fields Encrypted fields hash.
+      #
+      # @return [ Array <String> ] Array of auxiliary collections names.
+      def emm_collections(encrypted_fields)
+        [
+          encrypted_fields['escCollection'] || "enxcol_.#{name}.esc",
+          encrypted_fields['ecocCollection'] || "enxcol_.#{name}.ecoc",
+        ]
+      end
+
+      # Creating encrypted collections is only supported on 7.0.0 and later
+      # (wire version 21+).
+      #
+      # @param [ Mongo::Connection ] connection The connection to check
+      #   the wire version of.
+      #
+      # @raise [ Mongo::Error ] if the wire version is not
+      #   recent enough
+      def check_wire_version!(connection)
+        return unless connection.description.max_wire_version < QE2_MIN_WIRE_VERSION
+
+        raise Mongo::Error,
+              'Driver support of Queryable Encryption is incompatible with server. ' \
+              'Upgrade server to use Queryable Encryption.'
+      end
+
+      # Tries to return the encrypted fields from the argument. If the argument
+      # is nil, tries to find the encrypted fields from the
+      # encrypted_fields_map.
+      #
+      # @param [ Hash | nil ] fields the encrypted fields
+      #
+      # @return [ Hash ] the encrypted fields
+      def encrypted_fields_from(fields)
+        fields ||
+          (encrypted_fields_map && encrypted_fields_map[namespace]) ||
+          {}
+      end
+
+      # Tries to return the encrypted fields from the {{encrypted_fields_map}}
+      # value, for the current namespace.
+      #
+      # @return [ Hash | nil ] the encrypted fields, if found
+      def encrypted_fields_for_drop_from_map
+        encrypted_fields_map[namespace] ||
+          database.list_collections(filter: { name: name })
+                  .first
+                  &.fetch(:options, {})
+                  &.fetch(:encryptedFields, {}) ||
+          {}
+      end
+
+      # Returns a new create operation for the given collection.
+      #
+      # @param [ String ] coll the name of the collection to create.
+      #
+      # @return [ Operation::Create ] the new create operation.
+      def create_operation_for(coll)
+        Operation::Create.new(
+          selector: {
+            create: coll,
+            clusteredIndex: {
+              key: { _id: 1 },
+              unique: true
+            }
+          },
+          db_name: database.name
+        )
+      end
+    end
+  end
+end

data/lib/mongo/collection/view/aggregation.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2014-2020 MongoDB Inc.
 #
@@ -73,10 +73,33 @@ module Mongo
         # @param [ Array<Hash> ] pipeline The pipeline of operations.
         # @param [ Hash ] options The aggregation options.
         #
+        # @option options [ true, false ] :allow_disk_use Set to true if disk
+        #   usage is allowed during the aggregation.
+        # @option options [ Integer ] :batch_size The number of documents to return
+        #   per batch.
+        # @option options [ true, false ] :bypass_document_validation Whether or
+        #   not to skip document level validation.
+        # @option options [ Hash ] :collation The collation to use.
+        # @option options [ Object ] :comment A user-provided
+        #   comment to attach to this command.
+        # @option options [ String ] :hint The index to use for the aggregation.
+        # @option options [ Hash ] :let Mapping of variables to use in the pipeline.
+        #   See the server documentation for details.
+        # @option options [ Integer ] :max_time_ms The maximum amount of time in
+        #   milliseconds to allow the aggregation to run.
+        # @option options [ true, false ] :use_cursor Indicates whether the command
+        #   will request that the server provide results using a cursor. Note that
+        #   as of server version 3.6, aggregations always provide results using a
+        #   cursor and this option is therefore not valid.
+        # @option options [ Session ] :session The session to use.
+        #
         # @since 2.0.0
         def initialize(view, pipeline, options = {})
           @view = view
           @pipeline = pipeline.dup
+          unless Mongo.broken_view_aggregate || view.filter.empty?
+            @pipeline.unshift(:$match => view.filter)
+          end
           @options = BSON::Document.new(options).freeze
         end
 
@@ -108,37 +131,65 @@ module Mongo
           @view.send(:server_selector)
         end
 
-        def aggregate_spec(session)
-          Builder::Aggregation.new(pipeline, view, options.merge(session: session)).specification
+        def aggregate_spec(session, read_preference)
+          Builder::Aggregation.new(
+            pipeline,
+            view,
+            options.merge(session: session, read_preference: read_preference)
+          ).specification
         end
 
         def new(options)
           Aggregation.new(view, pipeline, options)
         end
 
-        def initial_query_op(session)
-          Operation::Aggregate.new(aggregate_spec(session))
+        def initial_query_op(session, read_preference)
+          Operation::Aggregate.new(aggregate_spec(session, read_preference))
         end
 
-        def valid_server?(server)
-          if secondary_ok?
-            true
+        # Return effective read preference for the operation.
+        #
+        # If the pipeline contains $merge or $out, and read preference specified
+        # by user is secondary or secondary_preferred, and target server is below
+        # 5.0, than this method returns primary read preference, because the
+        # aggregation will be routed to primary. Otherwise return the original
+        # read preference.
+        #
+        # See https://github.com/mongodb/specifications/blob/master/source/crud/crud.rst#read-preferences-and-server-selection
+        #
+        # @param [ Server::Connection ] connection The connection which
+        #   will be used for the operation.
+        # @return [ Hash | nil ] read preference hash that should be sent with
+        #   this command.
+        def effective_read_preference(connection)
+          return unless view.read_preference
+          return view.read_preference unless write?
+          return view.read_preference unless [:secondary, :secondary_preferred].include?(view.read_preference[:mode])
+
+          primary_read_preference = {mode: :primary}
+          description = connection.description
+          if description.primary?
+            log_warn("Routing the Aggregation operation to the primary server")
+            primary_read_preference
+          elsif description.mongos? && !description.features.merge_out_on_secondary_enabled?
+            log_warn("Routing the Aggregation operation to the primary server")
+            primary_read_preference
           else
-            description = server.description
-            description.standalone? || description.mongos? || description.primary? || description.load_balancer?
+            view.read_preference
           end
-        end
 
-        def secondary_ok?
-          !write?
         end
 
         def send_initial_query(server, session)
-          unless valid_server?(server)
-            log_warn("Rerouting the Aggregation operation to the primary server - #{server.summary} is not suitable")
-            server = cluster.next_primary(nil, session)
+          server.with_connection do |connection|
+            initial_query_op(
+              session,
+              effective_read_preference(connection)
+            ).execute_with_connection(
+              connection,
+              context: Operation::Context.new(client: client, session: session)
+            )
           end
-          initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session))
         end
 
         # Skip, sort, limit, projection are specified as pipeline stages

data/lib/mongo/collection/view/builder/aggregation.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
@@ -30,16 +30,17 @@ module Mongo
           #
           # @since 2.2.0
           MAPPINGS = BSON::Document.new(
-            :allow_disk_use => 'allowDiskUse',
-            :max_time_ms => 'maxTimeMS',
+            allow_disk_use: 'allowDiskUse',
+            bypass_document_validation: 'bypassDocumentValidation',
+            explain: 'explain',
+            collation: 'collation',
+            comment: 'comment',
+            hint: 'hint',
+            let: 'let',
             # This is intentional; max_await_time_ms is an alias for maxTimeMS
             # used on getMore commands for change streams.
-            :max_await_time_ms => 'maxTimeMS',
-            :explain => 'explain',
-            :bypass_document_validation => 'bypassDocumentValidation',
-            :collation => 'collation',
-            :hint => 'hint',
-            :comment => 'comment'
+            max_await_time_ms: 'maxTimeMS',
+            max_time_ms: 'maxTimeMS',
           ).freeze
 
           def_delegators :@view, :collection, :database, :read, :write_concern
@@ -55,12 +56,9 @@ module Mongo
 
           # Initialize the builder.
           #
-          # @example Initialize the builder.
-          #   Aggregation.new(map, reduce, view, options)
-          #
           # @param [ Array<Hash> ] pipeline The aggregation pipeline.
           # @param [ Collection::View ] view The collection view.
-          # @param [ Hash ] options The map/reduce options.
+          # @param [ Hash ] options The map/reduce and read preference options.
           #
           # @since 2.2.0
           def initialize(pipeline, view, options)
@@ -81,7 +79,7 @@ module Mongo
             spec = {
               selector: aggregation_command,
               db_name: database.name,
-              read: view.read_preference,
+              read: @options[:read_preference] || view.read_preference,
               session: @options[:session],
               collation: @options[:collation],
             }

data/lib/mongo/collection/view/builder/map_reduce.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2015-2020 MongoDB Inc.
 #
@@ -114,7 +114,7 @@ module Mongo
                 collection.read_concern)
             end
             command.update(view_options)
-            command.update(Utils.slice_hash(options, :collation))
+            command.update(options.slice(:collation))
 
             # Read preference isn't simply passed in the command payload
             # (it may need to be converted to wire protocol flags).

data/lib/mongo/collection/view/builder.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2015-2020 MongoDB Inc.
 #

data/lib/mongo/collection/view/change_stream/retryable.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# encoding: utf-8
+# rubocop:todo all
 
 # Copyright (C) 2017-2020 MongoDB Inc.
 #