mongo 2.11.6 → 2.12.0.rc0

data/lib/mongo/client_encryption.rb

@@ -0,0 +1,103 @@
+# Copyright (C) 2019 MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  # ClientEncryption encapsulates explicit operations on a key vault
+  # collection that cannot be done directly on a MongoClient. It
+  # provides an API for explicitly encrypting and decrypting values,
+  # and creating data keys.
+  class ClientEncryption
+    # Create a new ClientEncryption object with the provided options.
+    #
+    # @param [ Mongo::Client ] key_vault_client A Mongo::Client
+    #   that is connected to the MongoDB instance where the key vault
+    #   collection is stored.
+    # @param [ Hash ] options The ClientEncryption options.
+    #
+    # @option options [ String ] :key_vault_namespace The name of the
+    #   key vault collection in the format "database.collection".
+    # @option options [ Hash ] :kms_providers A hash of key management service
+    #   configuration information. Valid hash keys are :local or :aws. There
+    #   may be more than one KMS provider specified.
+    def initialize(key_vault_client, options={})
+      @encrypter = Crypt::ExplicitEncrypter.new(
+        key_vault_client,
+        options[:key_vault_namespace],
+        options[:kms_providers]
+      )
+    end
+
+    # Generates a data key used for encryption/decryption and stores
+    # that key in the KMS collection. The generated key is encrypted with
+    # the KMS master key.
+    #
+    # @param [ String ] kms_provider The KMS provider to use. Valid values are
+    #   "aws" and "local".
+    # @params [ Hash ] options
+    #
+    # @option options [ Hash ] :master_key Information about the AWS master key.
+    #   Required if kms_provider is "aws".
+    #   - :region [ String ] The The AWS region of the master key (required).
+    #   - :key [ String ] The Amazon Resource Name (ARN) of the master key (required).
+    #   - :endpoint [ String ] An alternate host to send KMS requests to (optional).
+    #     endpoint should be a host name with an optional port number separated
+    #     by a colon (e.g. "kms.us-east-1.amazonaws.com" or
+    #     "kms.us-east-1.amazonaws.com:443"). An endpoint in any other format
+    #     will not be properly parsed.
+    # @option options [ Array<String> ] :key_alt_names An optional array of
+    #   strings specifying alternate names for the new data key.
+    #
+    # @return [ BSON::Binary ] The 16-byte UUID of the new data key as a
+    #   BSON::Binary object with type :uuid.
+    def create_data_key(kms_provider, options={})
+      @encrypter.create_and_insert_data_key(
+        kms_provider,
+        options
+      )
+    end
+
+    # Encrypts a value using the specified encryption key and algorithm.
+    #
+    # @param [ Object ] value The value to encrypt.
+    # @param [ Hash ] options
+    #
+    # @option options [ BSON::Binary ] :key_id A BSON::Binary object of type :uuid
+    #   representing the UUID of the encryption key as it is stored in the key
+    #   vault collection.
+    # @option options [ String ] :key_alt_name The alternate name for the
+    #   encryption key.
+    # @option options [ String ] :algorithm The algorithm used to encrypt the value.
+    #   Valid algorithms are "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+    #   or "AEAD_AES_256_CBC_HMAC_SHA_512-Random".
+    #
+    # @note The :key_id and :key_alt_name options are mutually exclusive. Only
+    #   one is required to perform explicit encryption.
+    #
+    # @return [ BSON::Binary ] A BSON Binary object of subtype 6 (ciphertext)
+    #   representing the encrypted value.
+    def encrypt(value, options={})
+      @encrypter.encrypt(value, options)
+    end
+
+    # Decrypts a value that has already been encrypted.
+    #
+    # @param [ BSON::Binary ] value A BSON Binary object of subtype 6 (ciphertext)
+    #   that will be decrypted.
+    #
+    # @return [ Object ] The decrypted value.
+    def decrypt(value)
+      @encrypter.decrypt(value)
+    end
+  end
+end

data/lib/mongo/cluster.rb

@@ -258,8 +258,8 @@ module Mongo
     # @return [ Object ] The cluster topology.
     attr_reader :topology
 
-    # @return [ Mongo::Server::AppMetadata ] The application metadata, used for connection
-    #   handshakes.
+    # @return [ Mongo::Server::AppMetadata ] The application metadata, used for
+    #   connection handshakes.
     #
     # @since 2.4.0
     attr_reader :app_metadata

data/lib/mongo/cluster/reapers/cursor_reaper.rb

@@ -76,10 +76,15 @@ module Mongo
       #
       # @since 2.3.0
       def register_cursor(id)
-        if id && id > 0
-          @mutex.synchronize do
-            @active_cursors << id
-          end
+        if id.nil?
+          raise ArgumentError, 'register_cursor called with nil cursor_id'
+        end
+        if id == 0
+          raise ArgumentError, 'register_cursor called with cursor_id=0'
+        end
+
+        @mutex.synchronize do
+          @active_cursors << id
         end
       end
 
@@ -94,6 +99,13 @@ module Mongo
       #
       # @since 2.3.0
       def unregister_cursor(id)
+        if id.nil?
+          raise ArgumentError, 'unregister_cursor called with nil cursor_id'
+        end
+        if id == 0
+          raise ArgumentError, 'unregister_cursor called with cursor_id=0'
+        end
+
         @mutex.synchronize do
           @active_cursors.delete(id)
         end
@@ -122,12 +134,12 @@ module Mongo
             if server.features.find_command_enabled?
               Cursor::Builder::KillCursorsCommand.update_cursors(op_spec, active_cursors_copy.to_a)
               if Cursor::Builder::KillCursorsCommand.get_cursors_list(op_spec).size > 0
-                Operation::KillCursors.new(op_spec).execute(server)
+                Operation::KillCursors.new(op_spec).execute(server, client: nil)
               end
             else
               Cursor::Builder::OpKillCursors.update_cursors(op_spec, active_cursors_copy.to_a)
               if Cursor::Builder::OpKillCursors.get_cursors_list(op_spec).size > 0
-                Operation::KillCursors.new(op_spec).execute(server)
+                Operation::KillCursors.new(op_spec).execute(server, client: nil)
               end
             end
           end

data/lib/mongo/cluster/sdam_flow.rb

@@ -58,74 +58,26 @@ class Mongo::Cluster
     def update_server_descriptions
       servers_list.each do |server|
         if server.address == updated_desc.address
-          changed = server.description != updated_desc
+          @server_description_changed = server.description != updated_desc
+
           # Always update server description, so that fields that do not
           # affect description equality comparisons but are part of the
           # description are updated.
           server.update_description(updated_desc)
           server.update_last_scan
-          # But return if there was a content difference between
-          # descriptions, and if there wasn't we'll skip the remainder of
-          # sdam flow
-          return changed
+
+          # If there was no content difference between descriptions, we
+          # still need to run sdam flow, but if the flow produces no change
+          # in topology we will omit sending events.
+          return true
         end
       end
       false
     end
 
     def server_description_changed
-      if updated_desc.me_mismatch? && updated_desc.primary? &&
-        (topology.unknown? || topology.replica_set?)
-      then
-        # When the driver receives a description claiming to be a primary,
-        # we are obligated by spec tests to add and remove hosts in that
-        # description even if it also has a me mismatch. The me mismatch
-        # scenario though presents a number of problems:
-        #
-        # 1. Effectively, the server's address changes, meaning we cannot
-        # update the description of the server whose description change we
-        # are processing (instead servers are added and removed), but we
-        # behave to an extent as if we are updating the description, which
-        # causes a bunch of awkwardness.
-        # 2. The server for which we are processing the response will be
-        # removed from topology, which may cause the current thread to terminate
-        # prior to running the entire sdam flow. To deal with this we separate
-        # the removal event publication from actually removing the server
-        # from topology, which again complicates the flow.
-
-        # Primary-with-me-mismatch response could be the first one we receive
-        # when the topology is still unknown. Change to RS without primary
-        # in this case.
-        if topology.unknown?
-          @topology = Topology::ReplicaSetNoPrimary.new(
-            topology.options.merge(replica_set_name: updated_desc.replica_set_name),
-            topology.monitoring, self)
-        end
-
-        servers = add_servers_from_desc(updated_desc)
-        # Spec tests require us to remove servers based on data in descrptions
-        # with me mismatches. The driver will be more resilient if it only
-        # removed servers from descriptions with matching mes.
-        remove_servers_not_in_desc(updated_desc)
-
-        servers.each do |server|
-          server.start_monitoring
-        end
-
-        # The rest of sdam flow assumes the server being removed is not the one
-        # whose description we are processing, and publishes description update
-        # event. Since we are removing the server whose response we are
-        # processing, do not publish description change event but mark it
-        # published (by assigning to @previous_desc).
-        do_remove(updated_desc.address.to_s)
-        @previous_desc = updated_desc
-
-        # We may have removed the current primary, check if there is a primary.
-        check_if_has_primary
-        # Publish topology change event.
-        commit_changes
-        disconnect_servers
-        return
+      @previous_server_descriptions = servers_list.map do |server|
+        [server.address.to_s, server.description]
       end
 
       unless update_server_descriptions
@@ -158,10 +110,16 @@ class Mongo::Cluster
         end
       when Topology::Sharded
         unless updated_desc.unknown? || updated_desc.mongos?
+          log_warn(
+            "Removing server #{updated_desc.address.to_s} because it is a #{updated_desc.server_type.to_s.upcase} and not a MONGOS"
+          )
           remove
         end
       when Topology::ReplicaSetWithPrimary
         if updated_desc.standalone? || updated_desc.mongos?
+          log_warn(
+            "Removing server #{updated_desc.address.to_s} because it is a #{updated_desc.server_type.to_s.upcase} and not a replica set member"
+          )
           remove
           check_if_has_primary
         elsif updated_desc.primary?
@@ -173,6 +131,9 @@ class Mongo::Cluster
         end
       when Topology::ReplicaSetNoPrimary
         if updated_desc.standalone? || updated_desc.mongos?
+          log_warn(
+            "Removing server #{updated_desc.address.to_s} because it is a #{updated_desc.server_type.to_s.upcase} and not a replica set member"
+          )
           remove
         elsif updated_desc.primary?
           # Here we change topology type to RS with primary, however
@@ -440,6 +401,16 @@ class Mongo::Cluster
     end
 
     def publish_description_change_event
+      # This method may be invoked when server description definitely changed
+      # but prior to the topology getting updated. Therefore we check both
+      # server description changes and overall topology changes. When this
+      # method is called at the end of SDAM flow as part of "commit changes"
+      # step, server description change is incorporated into the topology
+      # change.
+      unless @server_description_changed || topology_effectively_changed?
+        return
+      end
+
       # updated_desc here may not be the description we received from
       # the server - in case of a stale primary, the server reported itself
       # as being a primary but updated_desc here will be unknown.
@@ -492,7 +463,7 @@ class Mongo::Cluster
       start_pool_if_data_bearing
 
       topology_changed_event_published = false
-      if topology.object_id != cluster.topology.object_id || @need_topology_changed_event
+      if !topology.equal?(cluster.topology) || @need_topology_changed_event
         # We are about to publish topology changed event.
         # Recreate the topology instance to get its server descriptions
         # up to date.
@@ -517,6 +488,10 @@ class Mongo::Cluster
         return
       end
 
+      unless topology_effectively_changed?
+        return
+      end
+
       # If we are here, there has been a change in the server descriptions
       # in our topology, but topology class has not changed.
       # Publish the topology changed event and recreate the topology to
@@ -584,5 +559,26 @@ class Mongo::Cluster
     def became_unknown?
       updated_desc.unknown? && !original_desc.unknown?
     end
+
+    # Returns whether topology meaningfully changed as a result of running
+    # SDAM flow.
+    #
+    # The spec defines topology equality through equality of topology types
+    # and server descriptions in each topology; this definition is not usable
+    # by us because our topology objects do not hold server descriptions and
+    # are instead "live". Thus we have to store the full list of server
+    # descriptions at the beginning of SDAM flow and compare them to the
+    # current ones.
+    def topology_effectively_changed?
+      unless topology.equal?(cluster.topology)
+        return true
+      end
+
+      server_descriptions = servers_list.map do |server|
+        [server.address.to_s, server.description]
+      end
+
+      @previous_server_descriptions != server_descriptions
+    end
   end
 end

data/lib/mongo/cluster/srv_monitor.rb

@@ -87,7 +87,7 @@ module Mongo
         rescue Resolv::ResolvTimeout => e
           log_warn("SRV monitor: timed out trying to resolve hostname #{@srv_uri.query_hostname}: #{e.class}: #{e}")
           return
-        rescue Timeout::Error
+        rescue ::Timeout::Error
           log_warn("SRV monitor: timed out trying to resolve hostname #{@srv_uri.query_hostname} (timeout=#{timeout})")
           return
         rescue Resolv::ResolvError => e

data/lib/mongo/collection.rb

@@ -249,7 +249,7 @@ module Mongo
                                 db_name: database.name,
                                 write_concern: write_concern,
                                 session: session
-                                }).execute(server)
+                                }).execute(server, client: client)
       end
     end
 
@@ -275,7 +275,7 @@ module Mongo
                               db_name: database.name,
                               write_concern: write_concern,
                               session: session
-                              }).execute(next_primary(nil, session))
+                              }).execute(next_primary(nil, session), client: client)
       end
     rescue Error::OperationFailure => ex
       raise ex unless ex.message =~ /ns not found/
@@ -545,7 +545,7 @@ module Mongo
               :id_generator => client.options[:id_generator],
               :session => session,
               :txn_num => txn_num
-           ).execute(server)
+           ).execute(server, client: client)
         end
       end
     end

data/lib/mongo/collection/view.rb

@@ -119,7 +119,7 @@ module Mongo
       # @option options :comment [ String ] Associate a comment with the query.
       # @option options :batch_size [ Integer ] The number of docs to return in
       #   each response from MongoDB.
-      # @option options :fields [ Hash ] The fields to include or exclude in
+      # @option options :projection [ Hash ] The fields to include or exclude in
       #   returned docs.
       # @option options :hint [ Hash ] Override default index selection and force
       #   MongoDB to use a specific index for the query.

data/lib/mongo/collection/view/aggregation.rb

@@ -125,7 +125,7 @@ module Mongo
             server = cluster.next_primary(nil, session)
           end
           validate_collation!(server)
-          initial_query_op(session).execute(server)
+          initial_query_op(session).execute(server, client: client)
         end
 
         def validate_collation!(server)

data/lib/mongo/collection/view/change_stream.rb

@@ -194,12 +194,21 @@ module Mongo
         # @example Close the change stream.
         #   stream.close
         #
-        # @return [ nil ] nil.
+        # @note This method attempts to close the cursor used by the change
+        #   stream, which in turn closes the server-side change stream cursor.
+        #   This method ignores any errors that occur when closing the
+        #   server-side cursor.
+        #
+        # @return [ nil ] Always nil.
         #
         # @since 2.5.0
         def close
           unless closed?
-            begin; @cursor.send(:kill_cursors); rescue; end
+            begin
+              @cursor.close
+            rescue Error::OperationFailure
+              # ignore
+            end
             @cursor = nil
           end
         end
@@ -340,7 +349,7 @@ module Mongo
         end
 
         def send_initial_query(server, session)
-          initial_query_op(session).execute(server)
+          initial_query_op(session).execute(server, client: client)
         end
 
         def time_to_bson_timestamp(time)

data/lib/mongo/collection/view/iterable.rb

@@ -55,14 +55,23 @@ module Mongo
           end
         end
 
-        # Stop the iteration by sending a KillCursors command to the server.
+        # Cleans up resources associated with this query.
         #
-        # @example Stop the iteration.
-        #   view.close_query
+        # If there is a server cursor associated with this query, it is
+        # closed by sending a KillCursors command to the server.
+        #
+        # @note This method propagates any errors that occur when closing the
+        #   server-side cursor.
+        #
+        # @return [ nil ] Always nil.
+        #
+        # @raise [ Error::OperationFailure ] If the server cursor close fails.
         #
         # @since 2.1.0
         def close_query
-          @cursor.send(:kill_cursors) if @cursor && !@cursor.closed?
+          if @cursor
+            @cursor.close
+          end
         end
         alias :kill_cursors :close_query
 
@@ -86,7 +95,7 @@ module Mongo
 
         def send_initial_query(server, session = nil)
           validate_collation!(server, collation)
-          initial_query_op(server, session).execute(server)
+          initial_query_op(server, session).execute(server, client: client)
         end
       end
     end