mongo 2.15.0 → 2.18.1

data/lib/mongo/cluster.rb

@@ -159,6 +159,10 @@ module Mongo
       @sdam_flow_lock = Mutex.new
       Session::SessionPool.create(self)
 
+      if seeds.empty? && load_balanced?
+        raise ArgumentError, 'Load-balanced clusters with no seeds are prohibited'
+      end
+
       # The opening topology is always unknown with no servers.
       # https://github.com/mongodb/specifications/pull/388
       opening_topology = Topology::Unknown.new(options, monitoring, self)
@@ -171,7 +175,7 @@ module Mongo
       @seeds = seeds = seeds.uniq
       servers = seeds.map do |seed|
         # Server opening events must be sent after topology change events.
-        # Therefore separate server addition, done here before topoolgy change
+        # Therefore separate server addition, done here before topology change
         # event is published, from starting to monitor the server which is
         # done later.
         add(seed, monitor: false)
@@ -179,11 +183,20 @@ module Mongo
 
       if seeds.size >= 1
         # Recreate the topology to get the current server list into it
-        @topology = topology.class.new(topology.options, topology.monitoring, self)
-        publish_sdam_event(
-          Monitoring::TOPOLOGY_CHANGED,
-          Monitoring::Event::TopologyChanged.new(opening_topology, @topology)
-        )
+        recreate_topology(topology, opening_topology)
+      end
+
+      if load_balanced?
+        # We are required by the specifications to produce certain SDAM events
+        # when in load-balanced topology.
+        # These events don't make a lot of sense from the standpoint of the
+        # driver's SDAM implementation, nor from the standpoint of the
+        # driver's load balancer implementation.
+        # They are just required boilerplate.
+        #
+        # Note that this call must be done above the monitoring_io check
+        # because that short-circuits the rest of the constructor.
+        fabricate_lb_sdam_events_and_set_server_type
       end
 
       if options[:monitoring_io] == false
@@ -203,7 +216,7 @@ module Mongo
       @connected = true
 
       if options[:cleanup] != false
-        @cursor_reaper = CursorReaper.new
+        @cursor_reaper = CursorReaper.new(self)
         @socket_reaper = SocketReaper.new(self)
         @periodic_executor = PeriodicExecutor.new([
           @cursor_reaper, @socket_reaper,
@@ -212,73 +225,77 @@ module Mongo
         @periodic_executor.run!
       end
 
-      # Need to record start time prior to starting monitoring
-      start_monotime = Utils.monotonic_time
+      unless load_balanced?
+        # Need to record start time prior to starting monitoring
+        start_monotime = Utils.monotonic_time
 
-      servers.each do |server|
-        server.start_monitoring
-      end
-
-      if options[:scan] != false
-        server_selection_timeout = options[:server_selection_timeout] || ServerSelector::SERVER_SELECTION_TIMEOUT
-        # The server selection timeout can be very short especially in
-        # tests, when the client waits for a synchronous scan before
-        # starting server selection. Limiting the scan to server selection time
-        # then aborts the scan before it can process even local servers.
-        # Therefore, allow at least 3 seconds for the scan here.
-        if server_selection_timeout < 3
-          server_selection_timeout = 3
+        servers.each do |server|
+          server.start_monitoring
         end
-        deadline = start_monotime + server_selection_timeout
-        # Wait for the first scan of each server to complete, for
-        # backwards compatibility.
-        # If any servers are discovered during this SDAM round we are going to
-        # wait for these servers to also be queried, and so on, up to the
-        # server selection timeout or the 3 second minimum.
-        loop do
-          # Ensure we do not try to read the servers list while SDAM is running
-          servers = @sdam_flow_lock.synchronize do
-            servers_list.dup
-          end
-          if servers.all? { |server| server.last_scan_monotime && server.last_scan_monotime >= start_monotime }
-            break
-          end
-          if (time_remaining = deadline - Utils.monotonic_time) <= 0
-            break
+
+        if options[:scan] != false
+          server_selection_timeout = options[:server_selection_timeout] || ServerSelector::SERVER_SELECTION_TIMEOUT
+          # The server selection timeout can be very short especially in
+          # tests, when the client waits for a synchronous scan before
+          # starting server selection. Limiting the scan to server selection time
+          # then aborts the scan before it can process even local servers.
+          # Therefore, allow at least 3 seconds for the scan here.
+          if server_selection_timeout < 3
+            server_selection_timeout = 3
           end
-          log_debug("Waiting for up to #{'%.2f' % time_remaining} seconds for servers to be scanned: #{summary}")
-          # Since the semaphore may have been signaled between us checking
-          # the servers list above and the wait call below, we should not
-          # wait for the full remaining time - wait for up to 1 second, then
-          # recheck the state.
-          begin
-            server_selection_semaphore.wait([time_remaining, 1].min)
-          rescue ::Timeout::Error
-            # nothing
+          deadline = start_monotime + server_selection_timeout
+          # Wait for the first scan of each server to complete, for
+          # backwards compatibility.
+          # If any servers are discovered during this SDAM round we are going to
+          # wait for these servers to also be queried, and so on, up to the
+          # server selection timeout or the 3 second minimum.
+          loop do
+            # Ensure we do not try to read the servers list while SDAM is running
+            servers = @sdam_flow_lock.synchronize do
+              servers_list.dup
+            end
+            if servers.all? { |server| server.last_scan_monotime && server.last_scan_monotime >= start_monotime }
+              break
+            end
+            if (time_remaining = deadline - Utils.monotonic_time) <= 0
+              break
+            end
+            log_debug("Waiting for up to #{'%.2f' % time_remaining} seconds for servers to be scanned: #{summary}")
+            # Since the semaphore may have been signaled between us checking
+            # the servers list above and the wait call below, we should not
+            # wait for the full remaining time - wait for up to 1 second, then
+            # recheck the state.
+            begin
+              server_selection_semaphore.wait([time_remaining, 1].min)
+            rescue ::Timeout::Error
+              # nothing
+            end
           end
         end
-      end
 
-      start_stop_srv_monitor
+        start_stop_srv_monitor
+      end
     end
 
     # Create a cluster for the provided client, for use when we don't want the
     # client's original cluster instance to be the same.
     #
-    # @api private
-    #
     # @example Create a cluster for the client.
     #   Cluster.create(client)
     #
     # @param [ Client ] client The client to create on.
+    # @param [ Monitoring | nil ] monitoring. The monitoring instance to use
+    #   with the new cluster. If nil, a new instance of Monitoring will be
+    #   created.
     #
     # @return [ Cluster ] The cluster.
     #
     # @since 2.0.0
-    def self.create(client)
+    # @api private
+    def self.create(client, monitoring: nil)
       cluster = Cluster.new(
         client.cluster.addresses.map(&:to_s),
-        Monitoring.new,
+        monitoring || Monitoring.new,
         client.cluster_options,
       )
       client.instance_variable_set(:@cluster, cluster)
@@ -321,6 +338,14 @@ module Mongo
     def_delegators :topology, :replica_set?, :replica_set_name, :sharded?,
                    :single?, :unknown?
 
+    # Returns whether the cluster is configured to be in the load-balanced
+    # topology.
+    #
+    # @return [ true | false ] Whether the topology is load-balanced.
+    def load_balanced?
+      topology.is_a?(Topology::LoadBalanced)
+    end
+
     [:register_cursor, :schedule_kill_cursor, :unregister_cursor].each do |m|
       define_method(m) do |*args|
         if options[:cleanup] != false
@@ -599,9 +624,25 @@ module Mongo
     #   on 4.2+ servers).
     # @option aptions [ true | false ] :awaited Whether the updated description
     #   was a result of processing an awaited hello.
+    # @option options [ Object ] :service_id Change state for the specified
+    #   service id only.
     #
     # @api private
     def run_sdam_flow(previous_desc, updated_desc, options = {})
+      if load_balanced?
+        if updated_desc.config.empty?
+          unless options[:keep_connection_pool]
+            servers_list.each do |server|
+              # TODO should service id be taken out of updated_desc?
+              # We could also assert that
+              # options[:service_id] == updated_desc.service_id
+              server.clear_connection_pool(service_id: options[:service_id])
+            end
+          end
+        end
+        return
+      end
+
       @sdam_flow_lock.synchronize do
         flow = SdamFlow.new(self, previous_desc, updated_desc,
           awaited: options[:awaited])
@@ -782,8 +823,15 @@ module Mongo
     def add(host, add_options=nil)
       address = Address.new(host, options)
       if !addresses.include?(address)
-        server = Server.new(address, self, @monitoring, event_listeners, options.merge(
-          monitor: false))
+        opts = options.merge(monitor: false)
+        # Note that in a load-balanced topology, every server must be a
+        # load balancer (load_balancer: true is specified in the options)
+        # but this option isn't set here because we are required by the
+        # specifications to pretent the server started out as an unknown one
+        # and publish server description change event into the load balancer
+        # one. The actual correct description for this server will be set
+        # by the fabricate_lb_sdam_events_and_set_server_type method.
+        server = Server.new(address, self, @monitoring, event_listeners, opts)
         @update_lock.synchronize do
           # Need to recheck whether server is present in @servers, because
           # the previous check was not under a lock.
@@ -891,14 +939,28 @@ module Mongo
     # is connected to does not support sessions.
     #
     # Session support may change over time, for example due to servers in the
-    # deployment being upgraded or downgraded. If the client is currently not
-    # connected to any data bearing servers, this method considers the state
-    # of session support as of when the client was last connected to at
-    # least one server. If the client has never connected to any servers,
-    # the deployment is considered to not support sessions.
+    # deployment being upgraded or downgraded.  If the client isn't connected to
+    # any servers and doesn't find any servers
+    # for the duration of server selection timeout, this method will raise
+    # NoServerAvailable. This method is called from the operation execution flow,
+    # and if it raises NoServerAvailable the entire operation will fail
+    # with that exception, since the operation execution has waited for
+    # the server selection timeout for any server to become available
+    # (which would be a superset of the servers suitable for the operation being
+    # attempted) and none materialized.
+    #
+    # @raise [ Error::SessionsNotAvailable ] If the deployment that the driver
+    #   is connected to does not support sessions.
+    # @raise [ Error::NoServerAvailable ] If the client isn't connected to
+    #   any servers and doesn't find any servers for the duration of
+    #   server selection timeout.
     #
     # @api private
     def validate_session_support!
+      if topology.is_a?(Topology::LoadBalanced)
+        return
+      end
+
       @state_change_lock.synchronize do
         @sdam_flow_lock.synchronize do
           if topology.data_bearing_servers?
@@ -912,25 +974,11 @@ module Mongo
       # No data bearing servers known - perform server selection to try to
       # get a response from at least one of them, to return an accurate
       # assessment of whether sessions are currently supported.
-      begin
-        ServerSelector.get(mode: :primary_preferred).select_server(self)
-        @state_change_lock.synchronize do
-          @sdam_flow_lock.synchronize do
-            unless topology.logical_session_timeout
-              raise_sessions_not_supported
-            end
-          end
-        end
-      rescue Error::NoServerAvailable
-        # We haven't been able to contact any servers - use last known
-        # value for session support.
-        @state_change_lock.synchronize do
-          @sdam_flow_lock.synchronize do
-            @update_lock.synchronize do
-              unless @sessions_supported
-                raise_sessions_not_supported
-              end
-            end
+      ServerSelector.get(mode: :primary_preferred).select_server(self)
+      @state_change_lock.synchronize do
+        @sdam_flow_lock.synchronize do
+          unless topology.logical_session_timeout
+            raise_sessions_not_supported
           end
         end
       end
@@ -982,6 +1030,41 @@ module Mongo
       msg = "The deployment that the driver is connected to does not support sessions: #{reason}"
       raise Error::SessionsNotSupported, msg
     end
+
+    def fabricate_lb_sdam_events_and_set_server_type
+      # Although there is no monitoring connection in load balanced mode,
+      # we must emit the following series of SDAM events.
+      server = @servers.first
+      # We are guaranteed to have the server here.
+      server.publish_opening_event
+      server_desc = server.description
+      # This is where a load balancer actually gets its correct server
+      # description.
+      server.update_description(
+        Server::Description.new(server.address, {},
+          load_balancer: true,
+          force_load_balancer: options[:connect] == :load_balanced,
+        )
+      )
+      publish_sdam_event(
+        Monitoring::SERVER_DESCRIPTION_CHANGED,
+        Monitoring::Event::ServerDescriptionChanged.new(
+          server.address,
+          topology,
+          server_desc,
+          server.description
+        )
+      )
+      recreate_topology(topology, topology)
+    end
+
+    def recreate_topology(new_topology_template, previous_topology)
+      @topology = topology.class.new(new_topology_template.options, new_topology_template.monitoring, self)
+      publish_sdam_event(
+        Monitoring::TOPOLOGY_CHANGED,
+        Monitoring::Event::TopologyChanged.new(previous_topology, @topology)
+      )
+    end
   end
 end
 

data/lib/mongo/collection/helpers.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2022 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Collection
+    # This module contains helper methods collection class.
+    #
+    # @api private
+    module Helpers
+      # Executes drop operation and and ignores NamespaceNotFound error.
+      #
+      # @param [ Operation::Drop ] operation Drop operation to be executed.
+      # @param [ Session ] session Session to be use for execution.
+      # @param [ Operation::Context ] context Context to use for execution.
+      #
+      # @return [ Result ] The result of the execution.
+      def do_drop(operation, session, context)
+        operation.execute(next_primary(nil, session), context: context)
+      rescue Error::OperationFailure => ex
+        # NamespaceNotFound
+        if ex.code == 26 || ex.code.nil? && ex.message =~ /ns not found/
+          false
+        else
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/mongo/collection/queryable_encryption.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2022 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Collection
+    # This module contains methods for creating and dropping auxiliary collections
+    # for queryable encryption.
+    #
+    # @api private
+    module QueryableEncryption
+
+      # Creates auxiliary collections and indices for queryable encryption if necessary.
+      #
+      # @param [ Hash | nil ] encrypted_fields Encrypted fields hash that was
+      #   provided to `create` collection helper.
+      # @param [ Client ] client Mongo client to be used to create auxiliary collections.
+      # @param [ Session ] session Session to be used to create auxiliary collections.
+      #
+      # @return [ Result ] The result of provided block.
+      def maybe_create_qe_collections(encrypted_fields, client, session)
+        encrypted_fields = if encrypted_fields
+          encrypted_fields
+        elsif encrypted_fields_map
+          encrypted_fields_map[namespace] || {}
+        else
+          {}
+        end
+        if encrypted_fields.empty?
+          return yield
+        end
+
+        emm_collections(encrypted_fields).each do |coll|
+          context = Operation::Context.new(client: client, session: session)
+          Operation::Create.new(
+            selector: {
+              create: coll,
+              clusteredIndex: {
+                key: {
+                  _id: 1
+                },
+                unique: true
+              }
+            },
+            db_name: database.name,
+          ).execute(next_primary(nil, session), context: context)
+        end
+        yield(encrypted_fields).tap do |result|
+          indexes.create_one(__safeContent__: 1) if result
+        end
+      end
+
+      # Drops auxiliary collections and indices for queryable encryption if necessary.
+      #
+      # @param [ Hash | nil ] encrypted_fields Encrypted fields hash that was
+      #   provided to `create` collection helper.
+      # @param [ Client ] client Mongo client to be used to drop auxiliary collections.
+      # @param [ Session ] session Session to be used to drop auxiliary collections.
+      #
+      # @return [ Result ] The result of provided block.
+      def maybe_drop_emm_collections(encrypted_fields, client, session)
+        encrypted_fields = if encrypted_fields
+          encrypted_fields
+        elsif encrypted_fields_map
+          if encrypted_fields_map[namespace]
+            encrypted_fields_map[namespace]
+          else
+            database.list_collections(filter: { name: name })
+              .first
+              &.fetch(:options, {})
+              &.fetch(:encryptedFields, {}) || {}
+          end
+        else
+          {}
+        end
+        if encrypted_fields.empty?
+          return yield
+        end
+        emm_collections(encrypted_fields).each do |coll|
+          context = Operation::Context.new(client: client, session: session)
+          operation = Operation::Drop.new(
+            selector: { drop: coll },
+            db_name: database.name,
+            session: session,
+          )
+          do_drop(operation, session, context)
+        end
+        yield
+      end
+
+      private
+
+      # Checks if names for auxiliary collections are set and returns them,
+      # otherwise returns default names.
+      #
+      # @param [ Hash ] encrypted_fields Encrypted fields hash.
+      #
+      # @return [ Array <String> ] Array of auxiliary collections names.
+      def emm_collections(encrypted_fields)
+        [
+          encrypted_fields['escCollection'] || "enxcol_.#{name}.esc",
+          encrypted_fields['eccCollection'] || "enxcol_.#{name}.ecc",
+          encrypted_fields['ecocCollection'] || "enxcol_.#{name}.ecoc",
+        ]
+      end
+
+    end
+  end
+end

data/lib/mongo/collection/view/aggregation.rb

@@ -73,10 +73,33 @@ module Mongo
         # @param [ Array<Hash> ] pipeline The pipeline of operations.
         # @param [ Hash ] options The aggregation options.
         #
+        # @option options [ true, false ] :allow_disk_use Set to true if disk
+        #   usage is allowed during the aggregation.
+        # @option options [ Integer ] :batch_size The number of documents to return
+        #   per batch.
+        # @option options [ true, false ] :bypass_document_validation Whether or
+        #   not to skip document level validation.
+        # @option options [ Hash ] :collation The collation to use.
+        # @option options [ Object ] :comment A user-provided
+        #   comment to attach to this command.
+        # @option options [ String ] :hint The index to use for the aggregation.
+        # @option options [ Hash ] :let Mapping of variables to use in the pipeline.
+        #   See the server documentation for details.
+        # @option options [ Integer ] :max_time_ms The maximum amount of time in
+        #   milliseconds to allow the aggregation to run.
+        # @option options [ true, false ] :use_cursor Indicates whether the command
+        #   will request that the server provide results using a cursor. Note that
+        #   as of server version 3.6, aggregations always provide results using a
+        #   cursor and this option is therefore not valid.
+        # @option options [ Session ] :session The session to use.
+        #
         # @since 2.0.0
         def initialize(view, pipeline, options = {})
           @view = view
           @pipeline = pipeline.dup
+          unless Mongo.broken_view_aggregate || view.filter.empty?
+            @pipeline.unshift(:$match => view.filter)
+          end
           @options = BSON::Document.new(options).freeze
         end
 
@@ -108,41 +131,64 @@ module Mongo
           @view.send(:server_selector)
         end
 
-        def aggregate_spec(session)
-          Builder::Aggregation.new(pipeline, view, options.merge(session: session)).specification
+        def aggregate_spec(session, read_preference)
+          Builder::Aggregation.new(
+            pipeline,
+            view,
+            options.merge(session: session, read_preference: read_preference)
+          ).specification
         end
 
         def new(options)
           Aggregation.new(view, pipeline, options)
         end
 
-        def initial_query_op(session)
-          Operation::Aggregate.new(aggregate_spec(session))
+        def initial_query_op(session, read_preference)
+          Operation::Aggregate.new(aggregate_spec(session, read_preference))
         end
 
-        def valid_server?(server)
-          description = server.with_connection do |connection|
-            connection.description
+        # Return effective read preference for the operation.
+        #
+        # If the pipeline contains $merge or $out, and read preference specified
+        # by user is secondary or secondary_preferred, and target server is below
+        # 5.0, than this method returns primary read preference, because the
+        # aggregation will be routed to primary. Otherwise return the original
+        # read preference.
+        #
+        # See https://github.com/mongodb/specifications/blob/master/source/crud/crud.rst#read-preferences-and-server-selection
+        #
+        # @param [ Server::Connection ] connection The connection which
+        #   will be used for the operation.
+        # @return [ Hash | nil ] read preference hash that should be sent with
+        #   this command.
+        def effective_read_preference(connection)
+          return unless view.read_preference
+          return view.read_preference unless write?
+          return view.read_preference unless [:secondary, :secondary_preferred].include?(view.read_preference[:mode])
+
+          primary_read_preference = {mode: :primary}
+          description = connection.description
+          if description.primary?
+            log_warn("Routing the Aggregation operation to the primary server")
+            primary_read_preference
+          elsif description.mongos? && !description.features.merge_out_on_secondary_enabled?
+            log_warn("Routing the Aggregation operation to the primary server")
+            primary_read_preference
+          else
+            view.read_preference
           end
-          description.standalone? || description.mongos? || description.primary? || secondary_ok?
-        end
 
-        def secondary_ok?
-          !write?
         end
 
         def send_initial_query(server, session)
-          unless valid_server?(server)
-            log_warn("Rerouting the Aggregation operation to the primary server - #{server.summary} is not suitable")
-            server = cluster.next_primary(nil, session)
-          end
-          validate_collation!(server)
-          initial_query_op(session).execute(server, context: Operation::Context.new(client: client, session: session))
-        end
-
-        def validate_collation!(server)
-          if options[:collation] && !server.with_connection { |connection| connection.features }.collation_enabled?
-            raise Error::UnsupportedCollation.new
+          server.with_connection do |connection|
+            initial_query_op(
+              session,
+              effective_read_preference(connection)
+            ).execute_with_connection(
+              connection,
+              context: Operation::Context.new(client: client, session: session)
+            )
           end
         end