mongo 2.17.4 → 2.18.0.beta1

data/lib/mongo/collection.rb

@@ -17,6 +17,8 @@
 
 require 'mongo/bulk_write'
 require 'mongo/collection/view'
+require 'mongo/collection/helpers'
+require 'mongo/collection/queryable_encryption'
 
 module Mongo
 
@@ -27,6 +29,8 @@ module Mongo
   class Collection
     extend Forwardable
     include Retryable
+    include QueryableEncryption
+    include Helpers
 
     # The capped option.
     #
@@ -47,8 +51,8 @@ module Mongo
     # @return [ Hash ] The collection options.
     attr_reader :options
 
-    # Get client, cluster, read preference, and write concern from client.
-    def_delegators :database, :client, :cluster
+    # Get client, cluster, read preference, write concern, and encrypted_fields_map from client.
+    def_delegators :database, :client, :cluster, :encrypted_fields_map
 
     # Delegate to the cluster for the next primary.
     def_delegators :cluster, :next_primary
@@ -58,10 +62,16 @@ module Mongo
     # @since 2.1.0
     CHANGEABLE_OPTIONS = [ :read, :read_concern, :write, :write_concern ].freeze
 
-    # Options that can be used for creating a time-series collection.
-    TIME_SERIES_OPTIONS = {
+    # Options map to transform create collection options.
+    #
+    # @api private
+    CREATE_COLLECTION_OPTIONS = {
       :time_series => :timeseries,
-      :expire_after => :expireAfterSeconds
+      :expire_after => :expireAfterSeconds,
+      :clustered_index => :clusteredIndex,
+      :change_stream_pre_and_post_images => :changeStreamPreAndPostImages,
+      :encrypted_fields => :encryptedFields,
+      :validator => :validator,
     }
 
     # Check if a collection is equal to another object. Will check the name and
@@ -94,10 +104,14 @@ module Mongo
     # @option options [ Hash ] :write_concern The write concern options.
     #   Can be :w => Integer|String, :fsync => Boolean, :j => Boolean.
     # @option options [ Hash ] :time_series Create a time-series collection.
-    #   See https://docs.mongodb.com/manual/core/timeseries-collections/ for more
+    #   See https://mongodb.com/docs/manual/core/timeseries-collections/ for more
     #   information about time-series collection.
     # @option options [ Integer ] :expire_after Number indicating
     #   after how many seconds old time-series data should be deleted.
+    # @options clustered_index [ Hash ] :clustered_index Create a clustered index.
+    #   This option specifies how this collection should be clustered on _id.
+    #   See https://www.mongodb.com/docs/v5.3/reference/method/db.createCollection/#std-label-db.createCollection.clusteredIndex
+    #   for more information about this option.
     #
     # @since 2.0.0
     def initialize(database, name, options = {})
@@ -241,6 +255,12 @@ module Mongo
     # @option opts [ Hash ] :time_series Create a time-series collection.
     # @option opts [ Integer ] :expire_after Number indicating
     #   after how many seconds old time-series data should be deleted.
+    # @option opts [ Hash ] :change_stream_pre_and_post_images Used to enable
+    #   pre- and post-images on the created collection.
+    # @option opts [ Hash ] :encrypted_fields Hash describing encrypted fields
+    #   for queryable encryption.
+    # @option opts [ Hash ] :validator Hash describing document validation
+    #   options for the collection.
     #
     # @return [ Result ] The result of the command.
     #
@@ -253,9 +273,9 @@ module Mongo
       options = Hash[self.options.reject do |key, value|
         %w(read read_preference read_concern).include?(key.to_s)
       end]
-      options.update(opts.slice(*TIME_SERIES_OPTIONS.keys))
-      # Converting Ruby spelled time series options to server style.
-      TIME_SERIES_OPTIONS.each do |ruby_key, server_key|
+      options.update(opts.slice(*CREATE_COLLECTION_OPTIONS.keys))
+      # Converting Ruby options to server style.
+      CREATE_COLLECTION_OPTIONS.each do |ruby_key, server_key|
         if options.key?(ruby_key)
           options[server_key] = options.delete(ruby_key)
         end
@@ -271,20 +291,24 @@ module Mongo
         end
 
         context = Operation::Context.new(client: client, session: session)
-        Operation::Create.new(
-          selector: operation,
-          db_name: database.name,
-          write_concern: write_concern,
-          session: session,
-          # Note that these are collection options, collation isn't
-          # taken from options passed to the create method.
-          collation: options[:collation] || options['collation'],
-        ).execute(next_primary(nil, session), context: context)
+        maybe_create_qe_collections(opts[:encrypted_fields], client, session) do |encrypted_fields|
+          Operation::Create.new(
+            selector: operation,
+            db_name: database.name,
+            write_concern: write_concern,
+            session: session,
+            # Note that these are collection options, collation isn't
+            # taken from options passed to the create method.
+            collation: options[:collation] || options['collation'],
+            encrypted_fields: encrypted_fields,
+            validator: options[:validator],
+          ).execute(next_primary(nil, session), context: context)
+        end
       end
     end
 
     # Drop the collection. Will also drop all indexes associated with the
-    # collection.
+    # collection, as well as associated queryable encryption collections.
     #
     # @note An error returned if the collection doesn't exist is suppressed.
     #
@@ -301,25 +325,22 @@ module Mongo
     # @since 2.0.0
     def drop(opts = {})
       client.send(:with_session, opts) do |session|
-        temp_write_concern = write_concern
-        write_concern = if opts[:write_concern]
-          WriteConcern.get(opts[:write_concern])
-        else
-          temp_write_concern
+        maybe_drop_emm_collections(opts[:encrypted_fields], client, session) do
+          temp_write_concern = write_concern
+          write_concern = if opts[:write_concern]
+            WriteConcern.get(opts[:write_concern])
+          else
+            temp_write_concern
+          end
+          context = Operation::Context.new(client: client, session: session)
+          operation = Operation::Drop.new({
+            selector: { :drop => name },
+            db_name: database.name,
+            write_concern: write_concern,
+            session: session,
+          })
+          do_drop(operation, session, context)
         end
-        Operation::Drop.new({
-                              selector: { :drop => name },
-                              db_name: database.name,
-                              write_concern: write_concern,
-                              session: session,
-                              }).execute(next_primary(nil, session), context: Operation::Context.new(client: client, session: session))
-      end
-    rescue Error::OperationFailure => ex
-      # NamespaceNotFound
-      if ex.code == 26 || ex.code.nil? && ex.message =~ /ns not found/
-        false
-      else
-        raise
       end
     end
 
@@ -343,7 +364,8 @@ module Mongo
     # @option options [ Integer ] :batch_size The number of documents returned in each batch
     #   of results from MongoDB.
     # @option options [ Hash ] :collation The collation to use.
-    # @option options [ String ] :comment Associate a comment with the query.
+    # @option options [ Object ] :comment A user-provided
+    #   comment to attach to this command.
     # @option options [ :tailable, :tailable_await ] :cursor_type The type of cursor to use.
     # @option options [ Integer ] :limit The max number of docs to return from the query.
     # @option options [ Integer ] :max_time_ms
@@ -386,7 +408,8 @@ module Mongo
     # @option options [ true, false ] :bypass_document_validation Whether or
     #   not to skip document level validation.
     # @option options [ Hash ] :collation The collation to use.
-    # @option options [ String ] :comment Associate a comment with the aggregation.
+    # @option options [ Object ] :comment A user-provided
+    #   comment to attach to this command.
     # @option options [ String ] :hint The index to use for the aggregation.
     # @option options [ Hash ] :let Mapping of variables to use in the pipeline.
     #   See the server documentation for details.
@@ -416,11 +439,35 @@ module Mongo
     # @param [ Array<Hash> ] pipeline Optional additional filter operators.
     # @param [ Hash ] options The change stream options.
     #
-    # @option options [ String ] :full_document Allowed values: ‘default’,
-    #   ‘updateLookup’. Defaults to ‘default’. When set to ‘updateLookup’,
-    #   the change notification for partial updates will include both a delta
-    #   describing the changes to the document, as well as a copy of the entire
-    #   document that was changed from some time after the change occurred.
+    # @option options [ String ] :full_document Allowed values: nil, 'default',
+    #   'updateLookup', 'whenAvailable', 'required'.
+    #
+    #   The default is to not send a value (i.e. nil), which is equivalent to
+    #   'default'. By default, the change notification for partial updates will
+    #   include a delta describing the changes to the document.
+    #
+    #   When set to 'updateLookup', the change notification for partial updates
+    #   will include both a delta describing the changes to the document as well
+    #   as a copy of the entire document that was changed from some time after
+    #   the change occurred.
+    #
+    #   When set to 'whenAvailable', configures the change stream to return the
+    #   post-image of the modified document for replace and update change events
+    #   if the post-image for this event is available.
+    #
+    #   When set to 'required', the same behavior as 'whenAvailable' except that
+    #   an error is raised if the post-image is not available.
+    # @option options [ String ] :full_document_before_change Allowed values: nil,
+    #   'whenAvailable', 'required', 'off'.
+    #
+    #   The default is to not send a value (i.e. nil), which is equivalent to 'off'.
+    #
+    #   When set to 'whenAvailable', configures the change stream to return the
+    #   pre-image of the modified document for replace, update, and delete change
+    #   events if it is available.
+    #
+    #   When set to 'required', the same behavior as 'whenAvailable' except that
+    #   an error is raised if the pre-image is not available.
     # @option options [ BSON::Document, Hash ] :resume_after Specifies the
     #   logical starting point for the new change stream.
     # @option options [ Integer ] :max_await_time_ms The maximum amount of time
@@ -433,6 +480,8 @@ module Mongo
     #   changes that occurred at or after the specified timestamp. Any command run
     #   against the server will return a cluster time that can be used here.
     #   Only recognized by server versions 4.0+.
+    # @option options [ Object ] :comment A user-provided
+    #   comment to attach to this command.
     #
     # @note A change stream only allows 'majority' read concern.
     # @note This helper method is preferable to running a raw aggregation with
@@ -460,6 +509,8 @@ module Mongo
     # @option options [ Hash ] :read The read preference options.
     # @option options [ Hash ] :collation The collation to use.
     # @option options [ Session ] :session The session to use.
+    # @option options [ Object ] :comment A user-provided
+    #   comment to attach to this command.
     #
     # @return [ Integer ] The document count.
     #
@@ -493,6 +544,9 @@ module Mongo
     #   command to run.
     # @option options :read [ Hash ] The read preference options.
     # @option options :collation [ Hash ] The collation to use.
+    # @option options [ Session ] :session The session to use.
+    # @option options [ Object ] :comment A user-provided
+    #   comment to attach to this command.
     #
     # @return [ Integer ] The document count.
     #
@@ -509,9 +563,11 @@ module Mongo
     #
     # @param [ Hash ] options Options for the operation.
     #
-    # @option opts :max_time_ms [ Integer ] The maximum amount of time to allow
+    # @option options :max_time_ms [ Integer ] The maximum amount of time to allow
     #   the command to run for on the server.
-    # @option opts [ Hash ] :read The read preference options.
+    # @option options [ Hash ] :read The read preference options.
+    # @option options [ Object ] :comment A user-provided
+    #   comment to attach to this command.
     #
     # @return [ Integer ] The document count.
     #
@@ -597,7 +653,8 @@ module Mongo
           raise ArgumentError, "Document to be inserted cannot be nil"
         end
 
-        write_with_retry(session, write_concern) do |server, txn_num|
+        context = Operation::Context.new(client: client, session: session)
+        write_with_retry(write_concern, context: context) do |connection, txn_num, context|
           Operation::Insert.new(
             :documents => [ document ],
             :db_name => database.name,
@@ -608,7 +665,8 @@ module Mongo
             :id_generator => client.options[:id_generator],
             :session => session,
             :txn_num => txn_num,
-          ).execute(server, context: Operation::Context.new(client: client, session: session))
+            :comment => opts[:comment]
+          ).execute_with_connection(connection, context: context)
         end
       end
     end
@@ -650,6 +708,8 @@ module Mongo
     # @option options [ true, false ] :bypass_document_validation Whether or
     #   not to skip document level validation.
     # @option options [ Session ] :session The session to use for the set of operations.
+    # @option options [ Hash ] :let Mapping of variables to use in the command.
+    #   See the server documentation for details.
     #
     # @return [ BulkWrite::Result ] The result of the operation.
     #

data/lib/mongo/config/options.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Mongo
+  module Config
+
+    # Encapsulates logic for setting options.
+    module Options
+
+      # Get the defaults or initialize a new empty hash.
+      #
+      # @return [ Hash ] The default options.
+      def defaults
+        @defaults ||= {}
+      end
+
+      # Define a configuration option with a default.
+      #
+      # @param [ Symbol ] name The name of the configuration option.
+      # @param [ Hash ] options Extras for the option.
+      #
+      # @option options [ Object ] :default The default value.
+      def option(name, options = {})
+        defaults[name] = settings[name] = options[:default]
+
+        class_eval do
+          # log_level accessor is defined specially below
+          define_method(name) do
+            settings[name]
+          end
+
+          define_method("#{name}=") do |value|
+            settings[name] = value
+          end
+
+          define_method("#{name}?") do
+            !!send(name)
+          end
+        end
+      end
+
+      # Reset the configuration options to the defaults.
+      #
+      # @example Reset the configuration options.
+      #   config.reset
+      #
+      # @return [ Hash ] The defaults.
+      def reset
+        settings.replace(defaults)
+      end
+
+      # Get the settings or initialize a new empty hash.
+      #
+      # @example Get the settings.
+      #   options.settings
+      #
+      # @return [ Hash ] The setting options.
+      def settings
+        @settings ||= {}
+      end
+    end
+  end
+end

data/lib/mongo/config/validators/option.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Mongo
+  module Config
+    module Validators
+
+      # Validator for configuration options.
+      #
+      # @api private
+      module Option
+        extend self
+
+        # Validate a configuration option.
+        #
+        # @example Validate a configuration option.
+        #
+        # @param [ String ] option The name of the option.
+        def validate(option)
+          unless Config.settings.keys.include?(option.to_sym)
+            raise Mongo::Error::InvalidConfigOption.new(option)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mongo/config.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "mongo/config/options"
+require "mongo/config/validators/option"
+
+module Mongo
+
+  # This module defines configuration options for Mongo.
+  #
+  # @api private
+  module Config
+    extend Forwardable
+    extend Options
+    extend self
+
+    option :validate_update_replace, default: false
+
+    # Set the configuration options.
+    #
+    # @example Set the options.
+    #   config.options = { validate_update_replace: true }
+    #
+    # @param [ Hash ] options The configuration options.
+    def options=(options)
+      options.each_pair do |option, value|
+        Validators::Option.validate(option)
+        send("#{option}=", value)
+      end
+    end
+  end
+end

data/lib/mongo/crypt/auto_encrypter.rb

@@ -32,6 +32,7 @@ module Mongo
 
       attr_reader :mongocryptd_client
       attr_reader :key_vault_client
+      attr_reader :metadata_client
       attr_reader :options
 
       # A Hash of default values for the :extra_options option
@@ -61,6 +62,22 @@ module Mongo
       #   auto-encryption. Default is false.
       # @option options [ Hash | nil ] :extra_options Options related to spawning
       #   mongocryptd. These are set to default values if no option is passed in.
+      # @option options [ Hash ] :kms_providers A hash of key management service
+      #   configuration information.
+      #   @see Mongo::Crypt::KMS::Credentials for list of options for every
+      #   supported provider.
+      #   @note There may be more than one KMS provider specified.
+      # @option options [ Hash ] :kms_tls_options TLS options to connect to KMS
+      #   providers. Keys of the hash should be KSM provider names; values
+      #   should be hashes of TLS connection options. The options are equivalent
+      #   to TLS connection options of Mongo::Client.
+      #   @see Mongo::Client#initialize for list of TLS options.
+      # @option options [ Hash | nil ] :encrypted_fields_map maps a collection
+      #   namespace to an encryptedFields.
+      #   - Note: If a collection is present on both the encryptedFieldsMap
+      #     and schemaMap, an error will be raised.
+      # @option options [ Boolean | nil ] :bypass_query_analysis When true
+      #   disables automatic analysis of outgoing commands.
       #
       # @raise [ ArgumentError ] If required options are missing or incorrectly
       #   formatted.
@@ -68,18 +85,20 @@ module Mongo
         @options = set_default_options(options).freeze
 
         @crypt_handle = Crypt::Handle.new(
-          @options[:kms_providers],
-          schema_map: @options[:schema_map]
+          Crypt::KMS::Credentials.new(@options[:kms_providers]),
+          Crypt::KMS::Validations.validate_tls_options(@options[:kms_tls_options]),
+          schema_map: @options[:schema_map],
+          encrypted_fields_map: @options[:encrypted_fields_map],
+          bypass_query_analysis: @options[:bypass_query_analysis]
         )
 
-        @key_vault_client = @options[:key_vault_client]
-
         # Set server selection timeout to 1 to prevent the client waiting for a
         # long timeout before spawning mongocryptd
         @mongocryptd_client = Client.new(
           @options[:extra_options][:mongocryptd_uri],
           monitoring_io: @options[:client].options[:monitoring_io],
           server_selection_timeout: 10,
+          database: @options[:client].options[:database]
         )
 
         begin
@@ -88,13 +107,14 @@ module Mongo
             mongocryptd_client: @mongocryptd_client,
             key_vault_namespace: @options[:key_vault_namespace],
             key_vault_client: @key_vault_client,
+            metadata_client: @metadata_client,
             mongocryptd_options: @options[:extra_options]
           )
         rescue
           begin
             @mongocryptd_client.close
           rescue => e
-            log_warn("Eror closing mongocryptd client in auto encrypter's constructor: #{e.class}: #{e}")
+            log_warn("Error closing mongocryptd client in auto encrypter's constructor: #{e.class}: #{e}")
             # Drop this exception so that the original exception is raised
           end
           raise
@@ -173,10 +193,63 @@ module Mongo
         end
 
         opts[:bypass_auto_encryption] ||= false
-        opts[:key_vault_client] ||= opts[:client]
+        set_or_create_clients(opts)
+        opts[:key_vault_client] = @key_vault_client
 
         Options::Redacted.new(opts).merge(extra_options: extra_options)
       end
+
+      # Create additional clients for auto encryption, if necessary
+      #
+      # @param [ Hash ] options Auto encryption options.
+      def set_or_create_clients(options)
+        client = options[:client]
+        @key_vault_client = if options[:key_vault_client]
+          options[:key_vault_client]
+        # https://jira.mongodb.org/browse/RUBY-3010
+        # https://jira.mongodb.org/browse/RUBY-3011
+        # Specification requires to use existing client when connection pool
+        # size is unlimited (0). Ruby driver does not support unlimited pool
+        # size.
+        # elsif client.options[:max_pool_size] == 0
+        #   client
+        else
+          internal_client(client)
+        end
+
+        @metadata_client = if options[:bypass_auto_encryption]
+          nil
+        # Specification requires to use existing client when connection pool
+        # size is unlimited (0). Ruby driver does not support unlimited pool
+        # size.
+        # https://jira.mongodb.org/browse/RUBY-3010
+        # https://jira.mongodb.org/browse/RUBY-3011
+        # elsif client.options[:max_pool_size] == 0
+        #   client
+        else
+          internal_client(client)
+        end
+      end
+
+      # Creates or return already created internal client to be used for
+      # auto encryption.
+      #
+      # @param [ Mongo::Client ] client  A client connected to the
+      #   encrypted collection.
+      #
+      # @return [ Mongo::Client ] Client to be used as internal client for
+      # auto encryption.
+      def internal_client(client)
+        @internal_client ||= client.with(
+          auto_encryption_options: nil,
+          # Specification requires that the internal client's connection pool
+          # size is unlimited (0). Ruby driver does not support unlimited pool
+          # size.
+          # https://jira.mongodb.org/browse/RUBY-3010
+          # https://jira.mongodb.org/browse/RUBY-3011
+          # max_pool_size: 0
+        )
+      end
     end
   end
 end