mongo 2.10.5 → 2.15.1

data/lib/mongo/client_encryption.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2019-2020 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  # ClientEncryption encapsulates explicit operations on a key vault
+  # collection that cannot be done directly on a MongoClient. It
+  # provides an API for explicitly encrypting and decrypting values,
+  # and creating data keys.
+  class ClientEncryption
+    # Create a new ClientEncryption object with the provided options.
+    #
+    # @param [ Mongo::Client ] key_vault_client A Mongo::Client
+    #   that is connected to the MongoDB instance where the key vault
+    #   collection is stored.
+    # @param [ Hash ] options The ClientEncryption options.
+    #
+    # @option options [ String ] :key_vault_namespace The name of the
+    #   key vault collection in the format "database.collection".
+    # @option options [ Hash ] :kms_providers A hash of key management service
+    #   configuration information. Valid hash keys are :local or :aws. There
+    #   may be more than one KMS provider specified.
+    def initialize(key_vault_client, options={})
+      @encrypter = Crypt::ExplicitEncrypter.new(
+        key_vault_client,
+        options[:key_vault_namespace],
+        options[:kms_providers]
+      )
+    end
+
+    # Generates a data key used for encryption/decryption and stores
+    # that key in the KMS collection. The generated key is encrypted with
+    # the KMS master key.
+    #
+    # @param [ String ] kms_provider The KMS provider to use. Valid values are
+    #   "aws" and "local".
+    # @param [ Hash ] options
+    #
+    # @option options [ Hash ] :master_key Information about the AWS master key.
+    #   Required if kms_provider is "aws".
+    #   - :region [ String ] The The AWS region of the master key (required).
+    #   - :key [ String ] The Amazon Resource Name (ARN) of the master key (required).
+    #   - :endpoint [ String ] An alternate host to send KMS requests to (optional).
+    #     endpoint should be a host name with an optional port number separated
+    #     by a colon (e.g. "kms.us-east-1.amazonaws.com" or
+    #     "kms.us-east-1.amazonaws.com:443"). An endpoint in any other format
+    #     will not be properly parsed.
+    # @option options [ Array<String> ] :key_alt_names An optional array of
+    #   strings specifying alternate names for the new data key.
+    #
+    # @return [ BSON::Binary ] The 16-byte UUID of the new data key as a
+    #   BSON::Binary object with type :uuid.
+    def create_data_key(kms_provider, options={})
+      @encrypter.create_and_insert_data_key(
+        kms_provider,
+        options
+      )
+    end
+
+    # Encrypts a value using the specified encryption key and algorithm.
+    #
+    # @param [ Object ] value The value to encrypt.
+    # @param [ Hash ] options
+    #
+    # @option options [ BSON::Binary ] :key_id A BSON::Binary object of type :uuid
+    #   representing the UUID of the encryption key as it is stored in the key
+    #   vault collection.
+    # @option options [ String ] :key_alt_name The alternate name for the
+    #   encryption key.
+    # @option options [ String ] :algorithm The algorithm used to encrypt the value.
+    #   Valid algorithms are "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+    #   or "AEAD_AES_256_CBC_HMAC_SHA_512-Random".
+    #
+    # @note The :key_id and :key_alt_name options are mutually exclusive. Only
+    #   one is required to perform explicit encryption.
+    #
+    # @return [ BSON::Binary ] A BSON Binary object of subtype 6 (ciphertext)
+    #   representing the encrypted value.
+    def encrypt(value, options={})
+      @encrypter.encrypt(value, options)
+    end
+
+    # Decrypts a value that has already been encrypted.
+    #
+    # @param [ BSON::Binary ] value A BSON Binary object of subtype 6 (ciphertext)
+    #   that will be decrypted.
+    #
+    # @return [ Object ] The decrypted value.
+    def decrypt(value)
+      @encrypter.decrypt(value)
+    end
+  end
+end

data/lib/mongo/cluster/periodic_executor.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,6 +25,7 @@ module Mongo
     #
     # @since 2.5.0
     class PeriodicExecutor
+      include BackgroundThread
 
       # The default time interval for the periodic executor to execute.
       #
@@ -31,47 +35,43 @@ module Mongo
       # Create a periodic executor.
       #
       # @example Create a PeriodicExecutor.
-      #   Mongo::Cluster::PeriodicExecutor.new(reaper, reaper2)
+      #   Mongo::Cluster::PeriodicExecutor.new([reaper, reaper2])
+      # @param [ Hash ] options The options.
+      #
+      # @option options [ Logger ] :logger A custom logger to use.
       #
       # @api private
       #
       # @since 2.5.0
-      def initialize(*executors)
+      def initialize(executors = [], options = {})
         @thread = nil
         @executors = executors
+        @stop_semaphore = Semaphore.new
+        @options = options
       end
 
-      # Start the thread.
-      #
-      # @example Start the periodic executor's thread.
-      #   periodic_executor.run!
-      #
-      # @api private
-      #
-      # @since 2.5.0
-      def run!
-        @thread && @thread.alive? ? @thread : start!
-      end
+      attr_reader :options
+
       alias :restart! :run!
 
-      # Stop the executor's thread.
-      #
-      # @example Stop the executors's thread.
-      #   periodic_executor.stop!
-      #
-      # @param [ Boolean ] wait Whether to wait for background threads to
-      #   finish running.
-      #
-      # @api private
-      #
-      # @since 2.5.0
-      def stop!(wait=false)
-        begin; flush; rescue; end
-        @thread.kill
-        if wait
-          @thread.join
+      def do_work
+        execute
+        @stop_semaphore.wait(FREQUENCY)
+      end
+
+      def pre_stop
+        @stop_semaphore.signal
+      end
+
+      def stop(final = false)
+        super
+
+        begin
+          flush
+        rescue
         end
-        !@thread.alive?
+
+        true
       end
 
       # Trigger an execute call on each reaper.
@@ -83,7 +83,8 @@ module Mongo
       #
       # @since 2.5.0
       def execute
-        @executors.each(&:execute) and true
+        @executors.each(&:execute)
+        true
       end
 
       # Execute all pending operations.
@@ -95,18 +96,8 @@ module Mongo
       #
       # @since 2.5.0
       def flush
-        @executors.each(&:flush) and true
-      end
-
-      private
-
-      def start!
-        @thread = Thread.new(FREQUENCY) do |i|
-          loop do
-            sleep(i)
-            execute
-          end
-        end
+        @executors.each(&:flush)
+        true
       end
     end
   end

data/lib/mongo/cluster/reapers/cursor_reaper.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2015 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -76,10 +79,15 @@ module Mongo
       #
       # @since 2.3.0
       def register_cursor(id)
-        if id && id > 0
-          @mutex.synchronize do
-            @active_cursors << id
-          end
+        if id.nil?
+          raise ArgumentError, 'register_cursor called with nil cursor_id'
+        end
+        if id == 0
+          raise ArgumentError, 'register_cursor called with cursor_id=0'
+        end
+
+        @mutex.synchronize do
+          @active_cursors << id
         end
       end
 
@@ -94,6 +102,13 @@ module Mongo
       #
       # @since 2.3.0
       def unregister_cursor(id)
+        if id.nil?
+          raise ArgumentError, 'unregister_cursor called with nil cursor_id'
+        end
+        if id == 0
+          raise ArgumentError, 'unregister_cursor called with cursor_id=0'
+        end
+
         @mutex.synchronize do
           @active_cursors.delete(id)
         end
@@ -118,16 +133,20 @@ module Mongo
         end
 
         to_kill_copy.each do |server, op_specs|
+          options = {
+            server_api: server.options[:server_api],
+          }
+          context = Operation::Context.new(options: options)
           op_specs.each do |op_spec|
             if server.features.find_command_enabled?
               Cursor::Builder::KillCursorsCommand.update_cursors(op_spec, active_cursors_copy.to_a)
               if Cursor::Builder::KillCursorsCommand.get_cursors_list(op_spec).size > 0
-                Operation::KillCursors.new(op_spec).execute(server)
+                Operation::KillCursors.new(op_spec).execute(server, context: context)
               end
             else
               Cursor::Builder::OpKillCursors.update_cursors(op_spec, active_cursors_copy.to_a)
               if Cursor::Builder::OpKillCursors.get_cursors_list(op_spec).size > 0
-                Operation::KillCursors.new(op_spec).execute(server)
+                Operation::KillCursors.new(op_spec).execute(server, context: context)
               end
             end
           end

data/lib/mongo/cluster/reapers/socket_reaper.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/cluster/sdam_flow.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2018-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2018-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,11 +28,13 @@ class Mongo::Cluster
   class SdamFlow
     extend Forwardable
 
-    def initialize(cluster, previous_desc, updated_desc)
+    def initialize(cluster, previous_desc, updated_desc, awaited: false)
       @cluster = cluster
       @topology = cluster.topology
-      @previous_desc = previous_desc
+      @original_desc = @previous_desc = previous_desc
       @updated_desc = updated_desc
+      @servers_to_disconnect = []
+      @awaited = !!awaited
     end
 
     attr_reader :cluster
@@ -48,6 +53,11 @@ class Mongo::Cluster
 
     attr_reader :previous_desc
     attr_reader :updated_desc
+    attr_reader :original_desc
+
+    def awaited?
+      @awaited
+    end
 
     def_delegators :topology, :replica_set_name
 
@@ -56,20 +66,35 @@ class Mongo::Cluster
     def update_server_descriptions
       servers_list.each do |server|
         if server.address == updated_desc.address
-          changed = server.description != updated_desc
-          # Always update server description, so that fields like
-          # last_update_time reflect the last ismaster response
+          # SDAM flow must be run when topology version in the new description
+          # is equal to the current topology version, per the example in
+          # https://github.com/mongodb/specifications/blob/master/source/server-discovery-and-monitoring/server-discovery-and-monitoring.rst#what-is-the-purpose-of-topologyversion
+          unless updated_desc.topology_version_gte?(server.description)
+            return false
+          end
+
+          @server_description_changed = server.description != updated_desc
+
+          # Always update server description, so that fields that do not
+          # affect description equality comparisons but are part of the
+          # description are updated.
           server.update_description(updated_desc)
-          # But return if there was a content difference between
-          # descriptions, and if there wasn't we'll skip the remainder of
-          # sdam flow
-          return changed
+          server.update_last_scan
+
+          # If there was no content difference between descriptions, we
+          # still need to run sdam flow, but if the flow produces no change
+          # in topology we will omit sending events.
+          return true
         end
       end
       false
     end
 
     def server_description_changed
+      @previous_server_descriptions = servers_list.map do |server|
+        [server.address.to_s, server.description]
+      end
+
       unless update_server_descriptions
         # All of the transitions require that server whose updated_desc we are
         # processing is still in the cluster (i.e., was not removed as a result
@@ -81,7 +106,16 @@ class Mongo::Cluster
 
       case topology
       when Topology::Single
-        # no changes ever
+        if topology.replica_set_name
+          if updated_desc.replica_set_name != topology.replica_set_name
+            log_warn(
+              "Server #{updated_desc.address.to_s} has an incorrect replica set name '#{updated_desc.replica_set_name}'; expected '#{topology.replica_set_name}'"
+            )
+            @updated_desc = ::Mongo::Server::Description.new(updated_desc.address,
+              {}, updated_desc.average_round_trip_time)
+            update_server_descriptions
+          end
+        end
       when Topology::Unknown
         if updated_desc.standalone?
           update_unknown_with_standalone
@@ -100,10 +134,16 @@ class Mongo::Cluster
         end
       when Topology::Sharded
         unless updated_desc.unknown? || updated_desc.mongos?
+          log_warn(
+            "Removing server #{updated_desc.address.to_s} because it is of the wrong type (#{updated_desc.server_type.to_s.upcase}) - expected SHARDED"
+          )
           remove
         end
       when Topology::ReplicaSetWithPrimary
         if updated_desc.standalone? || updated_desc.mongos?
+          log_warn(
+            "Removing server #{updated_desc.address.to_s} because it is of the wrong type (#{updated_desc.server_type.to_s.upcase}) - expected a replica set member"
+          )
           remove
           check_if_has_primary
         elsif updated_desc.primary?
@@ -115,6 +155,9 @@ class Mongo::Cluster
         end
       when Topology::ReplicaSetNoPrimary
         if updated_desc.standalone? || updated_desc.mongos?
+          log_warn(
+            "Removing server #{updated_desc.address.to_s} because it is of the wrong type (#{updated_desc.server_type.to_s.upcase}) - expected a replica set member"
+          )
           remove
         elsif updated_desc.primary?
           # Here we change topology type to RS with primary, however
@@ -138,7 +181,9 @@ class Mongo::Cluster
         raise ArgumentError, "Unknown topology #{topology.class}"
       end
 
+      verify_invariants
       commit_changes
+      disconnect_servers
     end
 
     # Transitions from unknown to single topology type, when a standalone
@@ -174,8 +219,8 @@ class Mongo::Cluster
       if topology.replica_set_name != updated_desc.replica_set_name
         log_warn(
           "Removing server #{updated_desc.address.to_s} because it has an " +
-          "incorrect replica set name (#{updated_desc.replica_set_name}); " +
-          "current set name is #{topology.replica_set_name}"
+          "incorrect replica set name '#{updated_desc.replica_set_name}'; " +
+          "expected '#{topology.replica_set_name}'"
         )
         remove
         check_if_has_primary
@@ -322,6 +367,9 @@ class Mongo::Cluster
           end
         end
       end
+
+      verify_invariants
+
       added_servers
     end
 
@@ -334,9 +382,14 @@ class Mongo::Cluster
       end.flatten
       servers_list.each do |server|
         unless updated_desc_address_strs.include?(address_str = server.address.to_s)
+          updated_host = updated_desc.address.to_s
+          if updated_desc.me && updated_desc.me != updated_host
+            updated_host += " (self-identified as #{updated_desc.me})"
+          end
           log_warn(
             "Removing server #{address_str} because it is not in hosts reported by primary " +
-            "#{updated_desc.address}"
+            "#{updated_host}. Reported hosts are: " +
+            updated_desc.hosts.join(', ')
           )
           do_remove(address_str)
         end
@@ -353,7 +406,21 @@ class Mongo::Cluster
     # Removes specified server from topology and warns if the topology ends
     # up with an empty server list as a result
     def do_remove(address_str)
-      cluster.remove(address_str)
+      servers = cluster.remove(address_str, disconnect: false)
+      servers.each do |server|
+        # We need to publish server closed event here, but we cannot close
+        # the server because it could be the server owning the monitor in
+        # whose thread this flow is presently executing, in which case closing
+        # the server can terminate the thread and leave SDAM processing
+        # incomplete. Thus we have to remove the server from the cluster,
+        # publish the event, but do not call disconnect on the server until
+        # the very end when all processing has completed.
+        publish_sdam_event(
+          Mongo::Monitoring::SERVER_CLOSED,
+          Mongo::Monitoring::Event::ServerClosed.new(server.address, cluster.topology)
+        )
+      end
+      @servers_to_disconnect += servers
       if servers_list.empty?
         log_warn(
           "Topology now has no servers - this is likely a misconfiguration of the cluster and/or the application"
@@ -362,19 +429,26 @@ class Mongo::Cluster
     end
 
     def publish_description_change_event
+      # This method may be invoked when server description definitely changed
+      # but prior to the topology getting updated. Therefore we check both
+      # server description changes and overall topology changes. When this
+      # method is called at the end of SDAM flow as part of "commit changes"
+      # step, server description change is incorporated into the topology
+      # change.
+      unless @server_description_changed || topology_effectively_changed?
+        return
+      end
+
       # updated_desc here may not be the description we received from
       # the server - in case of a stale primary, the server reported itself
       # as being a primary but updated_desc here will be unknown.
 
-      # We do not notify on unknown -> unknown changes.
-      # This can also be important for tests which have real i/o
-      # happening against bogus addresses which yield unknown responses
-      # and that also mock responses with the resulting race condition,
-      # though tests should avoid performing real i/o with monitoring_io: false
-      # option.
-      if updated_desc.unknown? && previous_desc.unknown?
-        return
-      end
+      # We used to not notify on Unknown -> Unknown server changes.
+      # Technically these are valid state changes (or at least as valid as
+      # other server description changes when the description has not
+      # changed meaningfully but the events are still published).
+      # The current version of the driver notifies on Unknown -> Unknown
+      # transitions.
 
       # Avoid dispatching events when updated description is the same as
       # previous description. This allows this method to be called multiple
@@ -391,6 +465,7 @@ class Mongo::Cluster
           topology,
           previous_desc,
           updated_desc,
+          awaited: awaited?,
         )
       )
       @previous_desc = updated_desc
@@ -411,9 +486,10 @@ class Mongo::Cluster
       # not all processed together.
 
       publish_description_change_event
+      start_pool_if_data_bearing
 
       topology_changed_event_published = false
-      if topology.object_id != cluster.topology.object_id || @need_topology_changed_event
+      if !topology.equal?(cluster.topology) || @need_topology_changed_event
         # We are about to publish topology changed event.
         # Recreate the topology instance to get its server descriptions
         # up to date.
@@ -438,6 +514,10 @@ class Mongo::Cluster
         return
       end
 
+      unless topology_effectively_changed?
+        return
+      end
+
       # If we are here, there has been a change in the server descriptions
       # in our topology, but topology class has not changed.
       # Publish the topology changed event and recreate the topology to
@@ -447,12 +527,33 @@ class Mongo::Cluster
       cluster.update_topology(topology)
     end
 
+    def disconnect_servers
+      while server = @servers_to_disconnect.shift
+        if server.connected?
+          # Do not publish server closed event, as this was already done
+          server.disconnect!
+        end
+      end
+    end
+
+    # If the server being processed is identified as data bearing, creates the
+    # server's connection pool so it can start populating
+    def start_pool_if_data_bearing
+      return if !updated_desc.data_bearing?
+
+      servers_list.each do |server|
+        if server.address == @updated_desc.address
+          server.pool
+        end
+      end
+    end
+
     # Checks if the cluster has a primary, and if not, transitions the topology
     # to ReplicaSetNoPrimary. Topology must be ReplicaSetWithPrimary when
     # invoking this method.
     def check_if_has_primary
       unless topology.replica_set?
-        raise ArgumentError, 'check_if_has_primary should only be called when topology is replica set'
+        raise ArgumentError, "check_if_has_primary should only be called when topology is replica set, but it is #{topology.class.name.sub(/.*::/, '')}"
       end
 
       primary = servers_list.detect do |server|
@@ -477,5 +578,43 @@ class Mongo::Cluster
       end
       false
     end
+
+    # Returns whether the server whose description this flow processed
+    # was not previously unknown, and is now. Used to decide, in particular,
+    # whether to clear the server's connection pool.
+    def became_unknown?
+      updated_desc.unknown? && !original_desc.unknown?
+    end
+
+    # Returns whether topology meaningfully changed as a result of running
+    # SDAM flow.
+    #
+    # The spec defines topology equality through equality of topology types
+    # and server descriptions in each topology; this definition is not usable
+    # by us because our topology objects do not hold server descriptions and
+    # are instead "live". Thus we have to store the full list of server
+    # descriptions at the beginning of SDAM flow and compare them to the
+    # current ones.
+    def topology_effectively_changed?
+      unless topology.equal?(cluster.topology)
+        return true
+      end
+
+      server_descriptions = servers_list.map do |server|
+        [server.address.to_s, server.description]
+      end
+
+      @previous_server_descriptions != server_descriptions
+    end
+
+    def verify_invariants
+      if Mongo::Lint.enabled?
+        if cluster.topology.single?
+          if cluster.servers_list.length > 1
+            raise Mongo::Error::LintError, "Trying to create a single topology with multiple servers: #{cluster.servers_list}"
+          end
+        end
+      end
+    end
   end
 end

data/lib/mongo/cluster/topology/base.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2018-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2018-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/cluster/topology/no_replica_set_options.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2018-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2018-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/lib/mongo/cluster/topology/replica_set_no_primary.rb

@@ -1,4 +1,7 @@
-# Copyright (C) 2014-2019 MongoDB, Inc.
+# frozen_string_literal: true
+# encoding: utf-8
+
+# Copyright (C) 2014-2020 MongoDB Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -73,8 +76,9 @@ module Mongo
         # @return [ true, false ] If a readable server is present.
         #
         # @since 2.4.0
+        # @deprecated
         def has_readable_server?(cluster, server_selector = nil)
-          (server_selector || ServerSelector.primary).candidates(cluster).any?
+          !(server_selector || ServerSelector.primary).try_select_server(cluster).nil?
         end
 
         # Determine if the topology would select a writable server for the
@@ -89,7 +93,7 @@ module Mongo
         #
         # @since 2.4.0
         def has_writable_server?(cluster)
-          cluster.servers.any?{ |server| server.primary? }
+          !ServerSelector.primary.try_select_server(cluster).nil?
         end
 
         # A replica set topology is a replica set.